test-tokenizer: reading a file and printing lexer tokens.

src/test-tokenizer.zig

@@ -0,0 +1,86 @@
+const std   = @import("std");
+const mem   = std.mem;
+const stdout = std.io.getStdOut().writer();
+const process = std.process;
+
+const fs = std.fs;
+
+const warn  = std.debug.warn;
+const print = std.debug.print;
+
+const lexer = @import("./tokenizer.zig");
+
+
+fn nextArg(args: [][]const u8, idx: *usize) ?[]const u8 {
+    if (idx.* >= args.len) return null;
+    defer idx.* += 1;
+    return args[idx.*];
+}
+
+
+// Get all tokens from the input.
+fn getAllTokens(allocator: *mem.Allocator, source: []const u8) !std.ArrayList(lexer.Token.Id) {
+
+    // Getting the tokenizer, initialized with the source code we want to check.
+    var tokenizer = lexer.Tokenizer.init(source);
+    var list = std.ArrayList(lexer.Token.Id).init(allocator);
+
+    while(true) {
+        const token = tokenizer.next();
+        try list.append(token.id);
+        if(token.id == .Eof) break;
+    }
+
+    return list;
+}
+
+pub fn get_file_size(path: []const u8) !u64 {
+    var file = try fs.cwd().openFile(path, .{});
+    defer file.close();
+
+    // Find the size of the file and create a buffer with this size.
+    var file_stat = try file.stat();
+    return file_stat.size;
+}
+
+pub fn lexer_analyze() !void {
+    // 1. get an allocator.
+    // 2. get the file path.
+    // 3. get the file size, and allocate file_size+1 bytes.
+    // 4. get the content of the file.
+    // 5. perform the analyze, and print each element.
+
+    // Create an allocator, for the arguments and the file.
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    defer arena.deinit();
+
+    const allocator = &arena.allocator;
+    var args = try process.argsAlloc(allocator);
+    defer process.argsFree(allocator, args);
+
+    // Get the file path.
+    var arg_idx: usize = 1; // Skipping the executable binary name.
+    const gui_file_path = nextArg(args, &arg_idx) orelse {
+        warn("Expected first argument to be path to gui file\n", .{});
+        return error.InvalidArgs;
+    };
+
+    // Get the file size.
+    const file_size = try get_file_size(gui_file_path);
+
+    // Get the file size and allocate memory.
+    const buffer = try allocator.alloc(u8, file_size + 1); // Last value will be a null-byte.
+    buffer[file_size] = 0;
+    const content = try fs.cwd().readFile(gui_file_path, buffer);
+    // print("file content is: {}", .{content}); // Working.
+
+    // Get the file size and allocate memory.
+    const tokens = try getAllTokens(allocator, content);
+    for(tokens.items) |tokenid| {
+        print("token: {}\n", .{@tagName(tokenid)});
+    }
+}
+
+pub fn main() !void {
+    try lexer_analyze();
+}