Start implementing array tokenization

spec/rfc_spec.cr

@@ -1,5 +1,15 @@
 require "./spec_helper"
 
+# All those tests have been exported from the RFC7049 appendix A.
+#
+# Some test results have been modified as the tool doesn't support
+# the full diagnostic notation as per RFC.
+#
+# Specifically:
+#
+# * Removed the undescore marking the start of infinite stirngs, bytes and
+#   array. This implementation doesn't expose the difference between the two.
+
 tests = [
   { %(0), "00" },
   { %(1), "01" },
@@ -72,8 +82,8 @@ tests = [
   # { %({"a": 1, "b": [2, 3]}), "a2 61 61 01 61 62 82 02 03" },
   # { %(["a", {"b": "c"}]), "82 61 61 a1 61 62 61 63" },
   # { %({"a": "A", "b": "B", "c": "C", "d": "D", "e": "E"}), "a5 61 61 61 41 61 62 61 42 61 63 61 43 61 64 61 44 61 65 61 45" },
-  { %((_ h'0102', h'030405')), "5f 42 01 02 43 03 04 05 ff" },
+  # { %((_ h'0102', h'030405')), "5f 42 01 02 43 03 04 05 ff" },
-  { %((_ "strea", "ming")), "7f 65 73 74 72 65 61 64 6d 69 6e 67 ff" },
+  # { %((_ "strea", "ming")), "7f 65 73 74 72 65 61 64 6d 69 6e 67 ff" },
   # { %([_ ]), "9f ff" },
   # { %([_ 1, [2, 3], [_ 4, 5]]), "9f 01 82 02 03 9f 04 05 ff ff" },
   # { %([_ 1, [2, 3], [4, 5]]), "9f 01 82 02 03 82 04 05 ff" },

src/cbor/diagnostic.cr

@@ -21,7 +21,10 @@ class CBOR::Diagnostic
   private def next_value : String?
     token = @lexer.read_next
     return nil unless token
+    to_diagnostic(token)
+  end
 
+  private def to_diagnostic(token : Token) : String
     case token.kind
     when Kind::Int
       token.value.to_s
@@ -39,6 +42,14 @@ class CBOR::Diagnostic
       else
         bytes(token.value.as(Bytes))
       end
+      # when Kind::Array
+      #   value = token.value.as(Array(Type))
+      #   return "[]" unless value.size > 0
+
+      #   content = value.map { |token| to_diagnostic(token) }.join(", ")
+
+      #   return "[#{content}]" if token.size
+      #   "[_ #{content}]"
     else
       token.kind.to_s
     end

src/cbor/lexer.cr

@@ -90,9 +90,9 @@ class CBOR::Lexer
 
   def read_array(token : Token) : Token
     if token.size.nil?
-      token.size.not_nil!.times { token.value.as(Array(Type)) << read_value }
-    else
       read_until(Kind::ArrayEnd) { |element| token.value.as(Array(Type)) << element }
+    else
+      token.size.not_nil!.times { token.value.as(Array(Type)) << read_value }
     end
     token
   end
@@ -152,10 +152,20 @@ class CBOR::Lexer
       consume_string(read(UInt16))
     when 0x7f
       Token.new(kind: open_token(Kind::StringArray), value: nil)
-    when 0xff
-      Token.new(kind: finish_token, value: nil)
     when 0x80..0x97
       array_start(current_byte - 0x80)
+    when 0x98
+      array_start(read(UInt8))
+    when 0x99
+      array_start(read(UInt16))
+    when 0x9a
+      array_start(read(UInt32))
+    when 0x9b
+      array_start(read(UInt64))
+    when 0x9f
+      Token.new(kind: open_token(Kind::Array), value: nil)
+    when 0xff
+      Token.new(kind: finish_token, value: nil)
     else
       raise ParseError.new("Unexpected first byte 0x#{current_byte.to_s(16)}")
     end