From 46bd32de2fff5bdbe04a9b0803695289694f7f13 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 9 Nov 2026 10:48:08 +0100 Subject: [PATCH] zml/tokenizer: grow the decoder output to fit any token IREE writes a token's text whole or not at all, so `feedOne` asserted whenever the caller's buffer was smaller than the token, e.g. Laguna's 415-byte whitespace tokens in a 128-byte buffer. The decoder now owns its output buffer and grows it the way `decode` already did, so `feedOne` and `finalize` no longer take a buffer. --- examples/llm/models/lfm2/session.zig | 7 +--- examples/llm/models/llama/session.zig | 7 +--- examples/llm/models/qwen3_5/session.zig | 7 +--- examples/llm/models/qwen3_5_moe/session.zig | 7 +--- zml/tokenizer/iree/tokenizer.zig | 42 ++++++++++----------- zml/tokenizer/tokenizer.zig | 12 +++--- 6 files changed, 33 insertions(+), 49 deletions(-) diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index ecc5abf8..86020cce 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -173,9 +173,6 @@ pub const Session = struct { var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); defer active_length_buf.deinit(); - const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); - const params = self.compiled_model.params; var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); @@ -185,7 +182,7 @@ pub const Session = struct { if (token_id == self.config.eos_token_id) break :generation; - const token = try decoder.feedOne(token_id, out_tokens_buffer); + const token = try decoder.feedOne(token_id); if (self.think_start) |think_start| if (token_id == think_start) { try stdout.writeAll("\x1b[2m"); }; @@ -221,7 +218,7 @@ pub const Session = struct { try current_token_buffer.toSlice(self.io, self.generated_token_slice); } - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.writeAll(try decoder.finalize()); try stdout.flush(); } }; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 8a6532e6..ce3aa4c4 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -172,9 +172,6 @@ pub const Session = struct { var decoder: zml.tokenizer.Tokenizer.Decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - const decoder_out_buffer: []u8 = try self.allocator.alloc(u8, 256); - defer self.allocator.free(decoder_out_buffer); - var last_token_id: u32 = self.last_generated_token; var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); @@ -188,7 +185,7 @@ pub const Session = struct { generation: while (true) { if (isEosToken(self.config, last_token_id)) break :generation; - try stdout.writeAll(try decoder.feedOne(last_token_id, decoder_out_buffer)); + try stdout.writeAll(try decoder.feedOne(last_token_id)); try stdout.flush(); try all_tokens.append(self.allocator, last_token_id); @@ -207,7 +204,7 @@ pub const Session = struct { last_token_id = try current_token_buffer.getValue(u32, self.io); } - try stdout.writeAll(try decoder.finalize(decoder_out_buffer)); + try stdout.writeAll(try decoder.finalize()); try stdout.flush(); } }; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index 5121fd30..28df02f3 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -138,9 +138,6 @@ pub const Session = struct { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); @@ -153,7 +150,7 @@ pub const Session = struct { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.eos_token_id) break :generation; - const token = try decoder.feedOne(token_id, out_tokens_buffer); + const token = try decoder.feedOne(token_id); if (self.think_start) |think_start| if (token_id == think_start) { try stdout.writeAll("\x1b[2m"); }; @@ -179,7 +176,7 @@ pub const Session = struct { try current_token_buffer.toSlice(self.io, self.generated_token_slice); } - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.writeAll(try decoder.finalize()); try stdout.flush(); } }; diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index 6a819f26..7b6c6cb2 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -147,9 +147,6 @@ pub const Session = struct { pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - const out_tokens_buffer = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); @@ -161,7 +158,7 @@ pub const Session = struct { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.eos_token_id) break :generation; - const token = try decoder.feedOne(token_id, out_tokens_buffer); + const token = try decoder.feedOne(token_id); if (self.think_start) |think_start| if (token_id == think_start) try stdout.writeAll("\x1b[2m"); try stdout.writeAll(token); if (self.think_end) |think_end| if (token_id == think_end) try stdout.writeAll("\x1b[0m"); @@ -183,7 +180,7 @@ pub const Session = struct { try current_token_buffer.toSlice(self.io, self.generated_token_slice); } - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.writeAll(try decoder.finalize()); try stdout.flush(); } }; diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index b1586afb..551e4bcb 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -380,6 +380,7 @@ pub const Tokenizer = struct { tokenizer: *const Tokenizer, state_storage: []u8, state: *c.iree_tokenizer_decode_state_t, + text: std.Io.Writer.Allocating, fn init(tokenizer: *const Tokenizer, allocator: std.mem.Allocator) !Decoder { var state_size: usize = undefined; @@ -399,12 +400,14 @@ pub const Tokenizer = struct { .tokenizer = tokenizer, .state_storage = state_storage, .state = state.?, + .text = .init(allocator), }; } pub fn deinit(self: *Decoder) void { c.iree_tokenizer_decode_state_deinitialize(self.state); self.tokenizer.allocator.free(self.state_storage); + self.text.deinit(); } pub fn reset(self: *Decoder) !void { @@ -419,33 +422,24 @@ pub const Tokenizer = struct { self.state = state.?; } - pub fn feedOne(self: *Decoder, token_id: u32, out: []u8) ![]u8 { - std.debug.assert(out.len != 0); - var consumed: usize = 0; - var produced: usize = 0; - try checkOk(c.iree_tokenizer_decode_state_feed( - self.state, - tokenIdList(&.{token_id}), - mutableStringView(out), - &consumed, - &produced, - )); - std.debug.assert(consumed == 1); - return out[0..produced]; + pub fn feedOne(self: *Decoder, token_id: u32) ![]const u8 { + self.text.clearRetainingCapacity(); + try self.feedInto(&.{token_id}, &self.text.writer); + return self.text.written(); } - pub fn finalize(self: *Decoder, out: []u8) ![]u8 { - std.debug.assert(out.len != 0); - var produced: usize = 0; - try checkOk(c.iree_tokenizer_decode_state_finalize( - self.state, - mutableStringView(out), - &produced, - )); - return out[0..produced]; + pub fn finalize(self: *Decoder) ![]const u8 { + self.text.clearRetainingCapacity(); + try self.finalizeInto(&self.text.writer); + return self.text.written(); } pub fn decode(self: *Decoder, token_ids: []const u32, out: *std.Io.Writer) !void { + try self.feedInto(token_ids, out); + try self.finalizeInto(out); + } + + fn feedInto(self: *Decoder, token_ids: []const u32, out: *std.Io.Writer) !void { var min_output: usize = @max(token_ids.len * 4, 128); var remaining = token_ids; while (remaining.len > 0) { @@ -480,8 +474,10 @@ pub const Tokenizer = struct { min_output = @max(128, min_output -| produced); remaining = remaining[consumed..]; } + } - min_output = 32; + fn finalizeInto(self: *Decoder, out: *std.Io.Writer) !void { + var min_output: usize = 32; while (true) { var produced: usize = 0; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index fc304b81..6d76a7b7 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -101,18 +101,18 @@ pub const Tokenizer = union(Tokenizers) { }; } - // Only valid until the next decode and while the decoder AND buffer is alive. - pub fn feedOne(self: *Decoder, token_id: u32, buffer: []u8) ![]const u8 { + // Only valid until the next decode and while the decoder is alive. + pub fn feedOne(self: *Decoder, token_id: u32) ![]const u8 { return switch (self.*) { - .iree => |*d| d.feedOne(token_id, buffer), + .iree => |*d| d.feedOne(token_id), inline else => |*d| try d.next(token_id) orelse &.{}, }; } - /// Output is only valid until the next feed/finalize call and while the decoder AND buffer is alive. - pub fn finalize(self: *Decoder, buffer: []u8) ![]const u8 { + /// Output is only valid until the next feed/finalize call and while the decoder is alive. + pub fn finalize(self: *Decoder) ![]const u8 { return switch (self.*) { - .iree => |*v| v.finalize(buffer), + .iree => |*v| v.finalize(), inline else => &.{}, }; }