zig bindings not working

[forrackun]

i was trying to implement examples/simple in zig but code is too slow and does not work. pretty sure i am doing something stupid. i need a good set of eyes. i am porting to zig cz i have other experiments in mind that i want to try.

const std = @import("std");

const c = @cImport({
    @cInclude("llama.h");
});

var gpa_instance = std.heap.GeneralPurposeAllocator(.{}){};
const gpa = gpa_instance.allocator();

pub fn main() !void {
    c.llama_backend_init(false);

    var lparams = c.llama_context_params{
        .seed = 1,
        .n_ctx = 512,
        .n_batch = 512,
        .n_gpu_layers = 0,
        .main_gpu = 0,
        .tensor_split = null,
        .rope_freq_base = 10000.0,
        .rope_freq_scale = 1.0,
        .progress_callback = null,
        .progress_callback_user_data = null,
        .low_vram = false,
        .f16_kv = false,
        .logits_all = false,
        .vocab_only = false,
        .use_mmap = true,
        .use_mlock = false,
        .embedding = false,
    };

    std.debug.print("Loading llama context with > \n {}", .{lparams});

    var model = c.llama_load_model_from_file("./models/orca-mini-3b.ggmlv3.q4_0.bin", lparams);
    if (model == null) {
        @panic("unreachable!");
    }

    var ctx = c.llama_new_context_with_model(model, lparams);
    if (ctx == null) {
        @panic("unreachable!");
    }

    std.debug.print("CTX {any} {any} \n", .{ ctx, @TypeOf(ctx) });

    var prompt = "Capital of Nepal is ";
    const prompt_size = prompt.len;
    const max_context_size = c.llama_n_ctx(ctx);

    std.debug.print("max_ctx_size {d} \n", .{max_context_size});

    var token_list = try std.ArrayList(c_int).initCapacity(gpa, @intCast(max_context_size));
    _ = token_list.addManyAsArrayAssumeCapacity(prompt_size);

    std.debug.print("token_list_size {d} \n", .{token_list.items.len});

    defer token_list.deinit();

    const n = c.llama_tokenize(ctx, @as([*]const u8, prompt.ptr), @as([*c]c_int, token_list.items.ptr), prompt_size, false);

    std.debug.print("model > \n {?} {d} \n", .{ model, n });

    std.debug.print("[PROMPT_START]\n", .{});

    var subtoken_list = token_list.items[0..@as(usize, @intCast(n))];

    for (subtoken_list) |token| {
        std.debug.print("{s}", .{c.llama_token_to_str(ctx, token)});
    }

    std.debug.print("\n[/PROMPT_END]\n", .{});

    try token_list.resize(@intCast(n));

    while (c.llama_get_kv_cache_token_count(ctx) < max_context_size) {
        if (c.llama_eval(ctx, token_list.items.ptr, @intCast(token_list.items.len), c.llama_get_kv_cache_token_count(ctx), 4) == 1) {
            @panic("Failed to eval");
        }

        std.debug.print("\n[AFTER_EVAL]\n", .{});

        var logits = c.llama_get_logits(ctx);
        var n_vocab = c.llama_n_vocab(ctx);

        var candidates = try gpa.alloc(c.llama_token_data, @intCast(n_vocab));

        for (0..@intCast(n_vocab)) |token_id| {
            var index = @as(usize, @intCast(token_id));
            var data = c.llama_token_data{ .id = @intCast(token_id), .logit = logits[index], .p = 0.0 };
            candidates[index] = data;

            // std.debug.print("\n[CANDIDATES] {n} {s}\n", .{ token_id, c.llama_token_to_str(ctx, logits[index]) });
        }

        var cdata = c.llama_token_data_array{
            .data = candidates.ptr,
            .size = candidates.len,
            .sorted = false,
        };

        var selected_token = c.llama_sample_token_greedy(ctx, &cdata);

        if (selected_token == c.llama_token_eos()) {
            std.debug.print("[EOS]\n", .{});
            break;
        }

        std.debug.print("{s}", .{c.llama_token_to_str(ctx, selected_token)});

        try token_list.append(selected_token);
        gpa.free(candidates);
    }

    c.llama_free(ctx);
    c.llama_free_model(model);
    c.llama_backend_free();
}

https://github.com/ggerganov/llama.cpp/blob/master/examples/simple/simple.cpp

[forrackun]

FULL CODE
https://github.com/forrackun/ppp/blob/main/adamixer.7z

MODEL

https://huggingface.co/TheBloke/orca_mini_3B-GGML/tree/main

[forrackun]

anyone please ?