diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index b1586afb..4b8ba1b3 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -419,6 +419,13 @@ pub const Tokenizer = struct { self.state = state.?; } + pub fn maxOutputLength(self: *const Decoder) usize { + // As IREE sizes its own decode buffer: "WordPiece can add 1 byte (space prefix), ByteLevel finalize can add + // 3 bytes (U+FFFD)." + // https://github.com/iree-org/iree/blob/3de6b83fba779c8b93e9cc9a10382eb25785cb22/runtime/src/iree/tokenizer/tokenizer.c#L694-L695 + return c.iree_tokenizer_vocab_max_token_length(c.iree_tokenizer_vocab(self.tokenizer.inner)) + 4; + } + pub fn feedOne(self: *Decoder, token_id: u32, out: []u8) ![]u8 { std.debug.assert(out.len != 0); var consumed: usize = 0; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index fc304b81..0673ff11 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -101,6 +101,14 @@ pub const Tokenizer = union(Tokenizers) { }; } + pub fn maxOutputLength(self: *const Decoder) usize { + return switch (self.*) { + .iree => |*d| d.maxOutputLength(), + // SentencePiece returns text from its own storage and never writes to the buffer. + inline else => 0, + }; + } + // Only valid until the next decode and while the decoder AND buffer is alive. pub fn feedOne(self: *Decoder, token_id: u32, buffer: []u8) ![]const u8 { return switch (self.*) {