diff --git a/src/test-tokenizer.zig b/src/test-tokenizer.zig new file mode 100644 index 0000000..0540eaa --- /dev/null +++ b/src/test-tokenizer.zig @@ -0,0 +1,86 @@ +const std = @import("std"); +const mem = std.mem; +const stdout = std.io.getStdOut().writer(); +const process = std.process; + +const fs = std.fs; + +const warn = std.debug.warn; +const print = std.debug.print; + +const lexer = @import("./tokenizer.zig"); + + +fn nextArg(args: [][]const u8, idx: *usize) ?[]const u8 { + if (idx.* >= args.len) return null; + defer idx.* += 1; + return args[idx.*]; +} + + +// Get all tokens from the input. +fn getAllTokens(allocator: *mem.Allocator, source: []const u8) !std.ArrayList(lexer.Token.Id) { + + // Getting the tokenizer, initialized with the source code we want to check. + var tokenizer = lexer.Tokenizer.init(source); + var list = std.ArrayList(lexer.Token.Id).init(allocator); + + while(true) { + const token = tokenizer.next(); + try list.append(token.id); + if(token.id == .Eof) break; + } + + return list; +} + +pub fn get_file_size(path: []const u8) !u64 { + var file = try fs.cwd().openFile(path, .{}); + defer file.close(); + + // Find the size of the file and create a buffer with this size. + var file_stat = try file.stat(); + return file_stat.size; +} + +pub fn lexer_analyze() !void { + // 1. get an allocator. + // 2. get the file path. + // 3. get the file size, and allocate file_size+1 bytes. + // 4. get the content of the file. + // 5. perform the analyze, and print each element. + + // Create an allocator, for the arguments and the file. + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); + + const allocator = &arena.allocator; + var args = try process.argsAlloc(allocator); + defer process.argsFree(allocator, args); + + // Get the file path. + var arg_idx: usize = 1; // Skipping the executable binary name. + const gui_file_path = nextArg(args, &arg_idx) orelse { + warn("Expected first argument to be path to gui file\n", .{}); + return error.InvalidArgs; + }; + + // Get the file size. + const file_size = try get_file_size(gui_file_path); + + // Get the file size and allocate memory. + const buffer = try allocator.alloc(u8, file_size + 1); // Last value will be a null-byte. + buffer[file_size] = 0; + const content = try fs.cwd().readFile(gui_file_path, buffer); + // print("file content is: {}", .{content}); // Working. + + // Get the file size and allocate memory. + const tokens = try getAllTokens(allocator, content); + for(tokens.items) |tokenid| { + print("token: {}\n", .{@tagName(tokenid)}); + } +} + +pub fn main() !void { + try lexer_analyze(); +}