From 9a9ed49c9dbec93fabd87aca77046aec05ba46a3 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 9 Nov 2026 13:24:56 +0100 Subject: [PATCH 1/2] zml/tokenizer: expose the decoder's maximum output length IREE writes a token's text whole or not at all, so `feedOne` needs a buffer that fits any token. `maxOutputLength` returns the bound IREE uses for its own decode buffer: the vocabulary's longest token plus 4 bytes. --- zml/tokenizer/iree/tokenizer.zig | 4 ++++ zml/tokenizer/tokenizer.zig | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index b1586afb..9ea25405 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -419,6 +419,10 @@ pub const Tokenizer = struct { self.state = state.?; } + pub fn maxOutputLength(self: *const Decoder) usize { + return c.iree_tokenizer_vocab_max_token_length(c.iree_tokenizer_vocab(self.tokenizer.inner)) + 4; + } + pub fn feedOne(self: *Decoder, token_id: u32, out: []u8) ![]u8 { std.debug.assert(out.len != 0); var consumed: usize = 0; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index fc304b81..b27ccdb0 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -101,6 +101,13 @@ pub const Tokenizer = union(Tokenizers) { }; } + pub fn maxOutputLength(self: *const Decoder) usize { + return switch (self.*) { + .iree => |*d| d.maxOutputLength(), + inline else => 0, + }; + } + // Only valid until the next decode and while the decoder AND buffer is alive. pub fn feedOne(self: *Decoder, token_id: u32, buffer: []u8) ![]const u8 { return switch (self.*) { From b5de0f167061726df3074dac94c315e3dd505304 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 9 Nov 2026 16:22:30 +0100 Subject: [PATCH 2/2] Add comments --- zml/tokenizer/iree/tokenizer.zig | 3 +++ zml/tokenizer/tokenizer.zig | 1 + 2 files changed, 4 insertions(+) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index 9ea25405..4b8ba1b3 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -420,6 +420,9 @@ pub const Tokenizer = struct { } pub fn maxOutputLength(self: *const Decoder) usize { + // As IREE sizes its own decode buffer: "WordPiece can add 1 byte (space prefix), ByteLevel finalize can add + // 3 bytes (U+FFFD)." + // https://github.com/iree-org/iree/blob/3de6b83fba779c8b93e9cc9a10382eb25785cb22/runtime/src/iree/tokenizer/tokenizer.c#L694-L695 return c.iree_tokenizer_vocab_max_token_length(c.iree_tokenizer_vocab(self.tokenizer.inner)) + 4; } diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index b27ccdb0..0673ff11 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -104,6 +104,7 @@ pub const Tokenizer = union(Tokenizers) { pub fn maxOutputLength(self: *const Decoder) usize { return switch (self.*) { .iree => |*d| d.maxOutputLength(), + // SentencePiece returns text from its own storage and never writes to the buffer. inline else => 0, }; }