Add support for terminating tokens

src/cbor/diagnostic.cr

@@ -6,7 +6,6 @@ require "./token"
 # provided in the RFC and ensuring a correct functioning of the `CBOR::Lexer`.
 class CBOR::Diagnostic
   @lexer : Lexer
-  @is_array : Bool = false
 
   def initialize(input)
     @lexer = Lexer.new(input)
@@ -29,17 +28,23 @@ class CBOR::Diagnostic
     return nil unless token
 
     case token
-    when Token::BytesArray
-      @is_array = true
-    when Token::BreakT
-      @is_array = flase
+    when Token::BytesArrayT
+      consume_bytes_array
+    else
+      Token.to_diagnostic(token)
+    end
+  end
+
+  private def consume_bytes_array : String
+    elements = [] of String
+
+    loop do
+      token = @lexer.next_token
+      raise "Unexpected EOF" unless token
+      break if token.is_a?(Token::BytesArrayEndT)
+      elements << Token.to_diagnostic(token)
     end
 
-    separator + Token.to_diagnostic(token)
-  end
-
-  private def separator : String
-    return ", " if @is_array
-    ""
+    "(_ #{elements.join(", ")})"
   end
 end

src/cbor/lexer.cr

@@ -1,8 +1,6 @@
 require "./token"
 
 class CBOR::Lexer
-  BREAK = 0xff
-
   def self.new(string : String)
     new IO::Memory.new(string)
   end
@@ -13,6 +11,10 @@ class CBOR::Lexer
 
   @current_pos : Int64
   @eof : Bool = false
+  # Holds a list of previously opened tokens.
+  # When a break in reached, the last entry in the array is
+  # the token to close.
+  @open_tokens = [] of Token::T
 
   def initialize(@io : IO)
     @current_pos = 0
@@ -60,10 +62,9 @@ class CBOR::Lexer
     when 0x5b
       consume_binary(read(UInt64))
     when 0x5f
-      Token::BytesArrayStartT.new(@current_pos)
+      open_token(Token::BytesArrayT.new(@current_pos))
     when 0xff
-      # TODO: Define which segment it's breaking
-      Token::BreakT.new(@current_pos)
+      finish_token
     else
       raise ParseError.new("Unexpected first byte 0x#{current_byte.to_s(16)}")
     end
@@ -89,6 +90,26 @@ class CBOR::Lexer
     Token::BytesT.new(@current_pos, bytes)
   end
 
+  private def open_token(token : Token::T) : Token::T
+    @open_tokens.push(token)
+    token
+  end
+
+  private def finish_token : Token::T
+    opened_token = @open_tokens.pop
+
+    case opened_token
+    when Token::ArrayT
+      Token::ArrayEndT.new(@current_pos)
+    when Token::BytesArrayT
+      Token::BytesArrayEndT.new(@current_pos)
+    when Token::StringArrayT
+      Token::StringArrayEndT.new(@current_pos)
+    else
+      raise ParseError.new("Unexpected token termination #{opened_token.class}")
+    end
+  end
+
   # Creates a method overloaded for each UInt sizes to convert the UInt into
   # the respective Int capable of containing the value
 

src/cbor/token.cr

@@ -3,28 +3,30 @@ class CBOR::Token
   record UndefinedT, byte_number : Int64
   record BoolT, byte_number : Int64, value : Bool
   record ArrayT, byte_number : Int64, size : UInt32?
+  record ArrayEndT, byte_number : Int64
   record MapT, byte_number : Int64, size : UInt32?
   record IntT, byte_number : Int64, value : Int8 | UInt8 | Int16 | UInt16 | Int32 | UInt32 | Int64 | UInt64 | Int128
   record FloatT, byte_number : Int64, value : Float64
   record StringT, byte_number : Int64, value : String
   record BytesT, byte_number : Int64, value : Bytes
-  record StringArrayStartT, byte_number : Int64
+  record StringArrayT, byte_number : Int64
   record StringArrayEndT, byte_number : Int64
-  record BytesArrayStartT, byte_number : Int64
+  record BytesArrayT, byte_number : Int64
   record BytesArrayEndT, byte_number : Int64
 
   alias T = NullT |
             UndefinedT |
             BoolT |
             ArrayT |
+            ArrayEndT |
             MapT |
             IntT |
             FloatT |
             StringT |
             BytesT |
-            StringArrayStartT |
+            StringArrayT |
             StringArrayEndT |
-            BytesArrayStartT |
+            BytesArrayT |
             BytesArrayEndT
 
   def self.to_diagnostic(token : T) : String
@@ -40,7 +42,7 @@ class CBOR::Token
       "undefined"
     when BoolT
       token.value.to_s
-    when BytesArrayStartT
+    when BytesArrayT
       "(_ "
     when BytesArrayEndT
       ")"