const std = @import("std"); const zml = @import("zml"); const inference = @import("inference.zig"); const model = @import("model.zig"); pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, compiled_model: *inference.CompiledModel, prefill: inference.KernelRunner, decode: inference.KernelRunner, config: *const model.Config, seqlen: u32, cache_buffers: zml.Bufferized(model.Cache), rng_buf: zml.Bufferized(zml.Tensor.Rng), generated_token_slice: zml.Slice, think_start: ?u32, think_end: ?u32, tokenizer: zml.tokenizer.Tokenizer, pub fn init( allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); errdefer model.Cache.unloadBuffers(&cache_buffers); var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); errdefer generated_token_slice.free(allocator); var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); errdefer prefill.deinit(allocator); const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); return .{ .allocator = allocator, .io = io, .platform = platform, .compiled_model = compiled_model, .prefill = prefill, .decode = decode, .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = compiled_model.params.seqlen, .cache_buffers = cache_buffers, .rng_buf = rng_buf, .generated_token_slice = generated_token_slice, .think_start = tokenizer.tokenId("") orelse unreachable, .think_end = tokenizer.tokenId("") orelse unreachable, }; } pub fn deinit(self: *Session) void { self.prefill.deinit(self.allocator); self.decode.deinit(self.allocator); model.Cache.unloadBuffers(&self.cache_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buf); self.generated_token_slice.free(self.allocator); } pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { var encoder = try self.tokenizer.encoder(); defer encoder.deinit(); const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, im_start }); const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); defer allocator.free(user_tokens); try tokens.appendSlice(allocator, user_tokens); const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); defer allocator.free(prompt_tokens); try tokens.appendSlice(allocator, prompt_tokens); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); defer allocator.free(assistant_tokens); try tokens.appendSlice(allocator, assistant_tokens); return tokens.toOwnedSlice(allocator); } pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { var encoder = try self.tokenizer.encoder(); defer encoder.deinit(); const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); defer allocator.free(user_tokens); try tokens.appendSlice(allocator, user_tokens); const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); defer allocator.free(prompt_tokens); try tokens.appendSlice(allocator, prompt_tokens); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); defer allocator.free(assistant_tokens); try tokens.appendSlice(allocator, assistant_tokens); return tokens.toOwnedSlice(allocator); } pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { const tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{ .batch = 1, .seq = self.seqlen }, .u32)); defer tokens_slice.free(self.allocator); const tokens = tokens_slice.items(u32); @memset(tokens, self.config.pad_token_id); @memcpy(tokens[0..all_tokens.len], all_tokens); var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); defer tokens_buf.deinit(); const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); defer tokens_pos_buf.deinit(); const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); defer active_length_buf.deinit(); const params = self.compiled_model.params; var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { .stablehlo => try params.attention_metadata.initBuffer(self.io, self.platform), .flash_attention => |metadata| .{ .flash_attention = switch (metadata) { .metal => .{ .metal = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, else => try metadata.initBuffer(self.io, self.platform), } }, }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); defer conv_cache_index_buffer.deinit(); var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); defer kv_cache_index_buffer.deinit(); var next_token_buf: zml.Buffer = undefined; inference.run(&self.prefill, .{ .io = self.io, .tokens_buf = &tokens_buf, .next_token_buf = &next_token_buf, .tokens_pos_buf = &tokens_pos_buf, .active_length_buf = &active_length_buf, .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, .attention_metadata_buffers = attention_metadata_buffers, }, &conv_cache_index_buffer, &kv_cache_index_buffer); defer next_token_buf.deinit(); try next_token_buf.toSlice(self.io, self.generated_token_slice); } pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{1})); var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); defer active_length_buf.deinit(); const params = self.compiled_model.params; var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.config.eos_token_id) break :generation; const token = try decoder.feedOne(token_id); if (self.think_start) |think_start| if (token_id == think_start) { try stdout.writeAll("\x1b[2m"); }; try stdout.writeAll(token); if (self.think_end) |think_end| if (token_id == think_end) { try stdout.writeAll("\x1b[0m"); }; try stdout.flush(); try all_tokens.append(self.allocator, token_id); if (all_tokens.items.len >= self.seqlen) break :generation; const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); defer token_pos_buffer.deinit(); var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); defer conv_cache_index_buffer.deinit(); var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); defer kv_cache_index_buffer.deinit(); inference.run(&self.decode, .{ .io = self.io, .tokens_buf = ¤t_token_buffer, .next_token_buf = ¤t_token_buffer, .tokens_pos_buf = &token_pos_buffer, .active_length_buf = &active_length_buf, .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, .attention_metadata_buffers = attention_metadata_buffers, }, &conv_cache_index_buffer, &kv_cache_index_buffer); try current_token_buffer.toSlice(self.io, self.generated_token_slice); } try stdout.writeAll(try decoder.finalize()); try stdout.flush(); } };