Start lexer and tests

spec/cbor/lexer_spec.cr

@@ -0,0 +1,35 @@
+require "../spec_helper"
+
+describe CBOR::Lexer do
+  describe "examples from the RFC7049 Appendix A" do
+    tests : Array(Tuple(String, Bytes)) = [
+      {"0", [0x00]},
+      {"1", [0x01]},
+      # {"10", "0a"},
+      # {"23", "17"},
+      # {"24", "18 18"},
+      # {"25", "18 19"},
+      # {"100", "18 64"},
+      # {"1000", "19 03 e8"},
+      # {"1000000", "1a 00 0f 42 40"},
+      # {"1000000000000", "1b 00 00 00 e8 d4 a5 10 00"},
+      # {"18446744073709551615", "1b ff ff ff ff ff ff ff ff"},
+      # {"18446744073709551616", "c2 49 01 00 00 00 00 00 00 0000"},
+      # {"-18446744073709551616", "3b ff ff ff ff ff ff ff ff"},
+      # {"-18446744073709551617", "c3 49 01 00 00 00 00 00 00 00 00"},
+      # {"-1", "20"},
+      # {"-10", "29"},
+      # {"-100", "38 63"},
+      # {"-1000", "39 03 e7"},
+    ]
+
+    tests.each do |tt|
+      it "Reads #{tt[1].inspect} as #{tt[0]}" do
+        lexer = CBOR::Lexer.new(tt[1])
+
+        token = lexer.read_token
+        CBOR::Token.to_s(token).should eq(tt[0])
+      end
+    end
+  end
+end

src/cbor/lexer.cr

@@ -1,67 +1,97 @@
 require "./token"
 
 class CBOR::Lexer
-  @current_byte : UInt8
+  def self.new(string : String)
+    new IO::Memory.new(string)
+  end
+
+  def self.new(slice : Bytes)
+    new IO::Memory.new(slice)
+  end
+
+  @token : Token::T
 
   def initialize(@io : IO)
-    @current_byte = 0x0
+    @byte_number = 0
+    @current_byte_number = 0
+    @token = Token::NullT.new(0)
+    @token_finished = true
   end
 
-  def next_token : Token?
+  @[AlwaysInline]
-    byte = @io.read_byte
+  def current_token : Token::T
-    return nil if byte.nil?
+    if @token_finished
+      @token_finished = false
+      @token = next_token
+    else
+      @token
+    end
+  end
 
-    @current_byte = byte
+  @[AlwaysInline]
+  def finish_token!
+    @token_finished = true
+  end
 
-    # See: RFC7049 Appedix B
+  @[AlwaysInline]
-    case @current_byte
+  def read_token : Token::T
-    when .<= 0x17
+    if @token_finished
-      Token.new(kind: Token::Kind::UInt, value: @curren_byte)
+      @token = next_token
+    else
+      finish_token!
+    end
+    @token
+  end
+
+  private def next_token
+    @current_byte_number = @byte_number
+    current_byte = next_byte
+
+    case current_byte
+    when 0x00..0x17
+      consume_int(current_byte)
     when 0x18
-      read_uint8
+      consume_int(read(Uint8))
     when 0x19
-      read_uint16
+      consume_int(read(Uint16))
     when 0x1a
-      read_uint32
+      consume_int(read(Uint32))
     when 0x1b
-      read_uint64
+      consume_int(read(Uint64))
-    when .<= 0x37
+    when 0x20..0x37
-      Token.new(kind: Token::Kind::NInt, value: Int8(@curren_byte))
+      consume_int(flip(current_byte.to_i8))
+    when 0x38
+      consume_int(flip(read(Uint8).to_i8))
+    when 0x39
+      consume_int(flip(read(Uint16).to_i16))
+    when 0x3a
+      consume_int(flip(read(Uint32).to_i32))
+    when 0x3b
+      consume_int(flip(read(Uint64).to_i64))
     end
   end
 
-  private def read_uint8
+  private def next_byte : Uint8
     byte = @io.read_byte
-    return unexpect_eof if byte.nil?
+    @byte_number += 1
-
+    fail unless byte
-    Token.new(kind: Token::Kind::UInt, value: byte)
+    byte
   end
 
-  private def read_uint16
+  private def consume_int(value)
-    value = UInt16.from_io(read_next(2), IO::ByteFormat::BigEndian)
+    Token::IntT.new(@current_byte_number, value)
-    Token.new(kind: Token::Kind::UInt, value: value)
   end
 
-  private def read_uint32
+  private def flip(value)
-    value = UInt32.from_io(read_next(4), IO::ByteFormat::BigEndian)
+    -1 - value
-    Token.new(kind: Token::Kind::UInt, value: value)
   end
 
-  private def read_uint64
+  private def read(type : T.class) forall T
-    value = UInt64.from_io(read_next(8), IO::ByteFormat::BigEndian)
+    @byte_number += sizeof(T)
-    Token.new(kind: Token::Kind::UInt, value: value)
+    @io.read_bytes(T, IO::ByteFormat::NetworkEndian)
   end
 
-  private def read_next(n : Int)
+  private def fail
-    slice = Bytes.new(n)
+    raise "Pase error"
-
-    read = @io.read(slice)
-    return unexpect_eof if read == 0
-
-    slice
-  end
-
-  private def unexpected_eof
-    raise "Unexpected EOF"
   end
 end

src/cbor/token.cr

@@ -1,14 +1,21 @@
-class CBOR::Token(T)
+class CBOR::Token
-  enum Kind
+  record NullT, byte_number : Int32
-    UInt8
+  record BoolT, byte_number : Int32, value : Bool
-    NInt
+  record ArrayT, byte_number : Int32, size : UInt32
-    Byte
+  record MapT, byte_number : Int32, size : UInt32
-    Text
+  record IntT, byte_number : Int32, value : Int8 | UInt8 | Int16 | UInt16 | Int32 | UInt32 | Int64 | UInt64
-    Array
+  record FloatT, byte_number : Int32, value : Float64
-    Map
+  record StringT, byte_number : Int32, value : String
-    Float
+  record BytesT, byte_number : Int32, value : Bytes
-  end
 
-  def initialize(@value : T, @kind : Kind)
+  alias T = NullT | BoolT | ArrayT | MapT | IntT | FloatT | StringT | BytesT
+
+  def self.to_s(token : T)
+    case token
+    when IntT
+      token.value.to_s
+    else
+      "NOT IMPLEMENTED YET!"
+    end
   end
 end