From 17f4fa2fb35d300fd4b8d529680b32c07de37bc5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 5 Jan 2026 11:58:21 +0100 Subject: [PATCH] Try something with Arguments.bake() --- examples/llama/main.zig | 4 +++- zml2/exe.zig | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6272dcaa..02069b86 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -409,6 +409,8 @@ pub fn generateText( var decode_args = try decode_exe.args(allocator); defer decode_args.deinit(allocator); + decode_args.bake(.{llama_buffers}); + var decode_results = try decode_exe.results(allocator); defer decode_results.deinit(allocator); @@ -447,7 +449,7 @@ pub fn generateText( defer token_pos_buffer.deinit(); // call to generate the next token - decode_args.set(.{ llama_buffers, current_token_buffer, token_pos_buffer, kv_cache_buffers, rng_buffers }); + decode_args.set(.{ current_token_buffer, token_pos_buffer, kv_cache_buffers, rng_buffers }); decode_exe.call(decode_args, &decode_results); diff --git a/zml2/exe.zig b/zml2/exe.zig index 122f99d8..fc7b5580 100644 --- a/zml2/exe.zig +++ b/zml2/exe.zig @@ -90,6 +90,7 @@ pub const Exe = struct { pub const Arguments = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, + baked_count: usize = 0, pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); @@ -118,7 +119,7 @@ pub const Exe = struct { self: *Arguments, current_index: usize = 0, }; - var context: LocalContext = .{ .self = self, .current_index = offset }; + var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const Buffer) void { stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); @@ -130,6 +131,25 @@ pub const Exe = struct { } }.cb, &context, &v); } + + pub fn bake(self: *Arguments, v: anytype) void { + const LocalContext = struct { + self: *Arguments, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = self.baked_count }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + self.baked_count = context.current_index; + } }; pub const Results = struct {