guid/src/test-tokenizer.zig

const std   = @import("std");
const mem   = std.mem;
const stdout = std.io.getStdOut().writer();
const process = std.process;

const fs = std.fs;

const warn  = std.debug.warn;
const print = std.debug.print;

const lexer = @import("./tokenizer.zig");


fn nextArg(args: [][]const u8, idx: *usize) ?[]const u8 {
    if (idx.* >= args.len) return null;
    defer idx.* += 1;
    return args[idx.*];
}


// Get all tokens from the input.
fn getAllTokens(allocator: *mem.Allocator, source: []const u8) !std.ArrayList(lexer.Token) {

    // Getting the tokenizer, initialized with the source code we want to check.
    var tokenizer = lexer.Tokenizer.init(source);
    var list = std.ArrayList(lexer.Token).init(allocator);

    while(true) {
        const token = tokenizer.next();
        try list.append(token);
        if(token.id == .Eof) break;
    }

    return list;
}

pub fn get_file_size(path: []const u8) !u64 {
    var file = try fs.cwd().openFile(path, .{});
    defer file.close();

    // Find the size of the file and create a buffer with this size.
    var file_stat = try file.stat();
    return file_stat.size;
}

pub fn lexer_analyze() !void {
    // 1. get an allocator.
    // 2. get the file path.
    // 3. get the file size, and allocate file_size+1 bytes.
    // 4. get the content of the file.
    // 5. perform the analyze, and print each element.

    // Create an allocator, for the arguments and the file.
    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
    defer arena.deinit();

    const allocator = &arena.allocator;
    var args = try process.argsAlloc(allocator);
    defer process.argsFree(allocator, args);

    // Get the file path.
    var arg_idx: usize = 1; // Skipping the executable binary name.
    const gui_file_path = nextArg(args, &arg_idx) orelse {
        warn("Expected first argument to be path to gui file\n", .{});
        return error.InvalidArgs;
    };

    // Get the file size.
    const file_size = try get_file_size(gui_file_path);

    // Get the file size and allocate memory.
    const buffer = try allocator.alloc(u8, file_size + 1); // Last value will be a null-byte.
    buffer[file_size] = 0;
    const content = try fs.cwd().readFile(gui_file_path, buffer);
    // print("file content is: {}", .{content}); // Working.

    // Get the file size and allocate memory.
    const tokens = try getAllTokens(allocator, content);
    for(tokens.items) |token| {
        print("{s:20} => {}\n", .{@tagName(token.id), buffer[token.loc.start..token.loc.end]});
    }
}

pub fn main() !void {
    try lexer_analyze();
}