diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index ea66f7a1..bb53bcb8 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -126,25 +126,19 @@ filegroup( ) use_repo(huggingface, "Meta-Llama-3.1-70B-Instruct") - -huggingface.model( - name = "TinyLlama-120M-scratch", - build_file_content = """\ -package(default_visibility = ["//visibility:public"]) -filegroup( - name = "TinyLlama-120M-scratch", - srcs = glob(["*.json", "*.safetensors"]), +http_file( + name = "Karpathy-TinyLlama-Stories15M", + downloaded_file_path = "stories15M.tinyllama", + sha256 = "cd590644d963867a2b6e5a1107f51fad663c41d79c149fbecbbb1f95fa81f49a", + url = "https://huggingface.co/karpathy/tinyllamas/resolve/0bd21da7698eaf29a0d7de3992de8a46ef624add/stories15M.bin?download=true", ) -""", - commit = "89c1bb4ea00861ddaa26c55f102ccb25e161feee", - includes = [ - "*.safetensors", - "*.json", - ], - model = "Hoyeon/TinyLlama-120M-scratch", -) -use_repo(huggingface, "TinyLlama-120M-scratch") +http_file( + name = "Karpathy-TinyLlama-Tokenizer", + downloaded_file_path = "stories260K.tinyllama", + sha256 = "50a52ef822ee9e83de5ce9d0be0a025a773d019437f58b5ff9dcafb063ece361", + url = "https://github.com/karpathy/llama2.c/raw/c02865df300f3bd9e567ce061000dc23bf785a17/tokenizer.bin", +) bazel_dep(name = "rules_rust", version = "0.57.0") rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 8d030233..5efbc68b 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -1,7 +1,8 @@ load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:native_binary.bzl", "native_binary") +load("@bazel_skylib//rules:native_binary.bzl", "native_test") +load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@zml//bazel:zig.bzl", "zig_cc_binary") @@ -20,24 +21,6 @@ zig_cc_binary( ], ) -cc_binary( - name = "TinyLlama-120M-scratch", - args = [ - "--config=$(location @TinyLlama-120M-scratch//:config.json)", - "--weights=$(location @TinyLlama-120M-scratch//:model.safetensors)", - "--tokenizer=$(location @TinyLlama-120M-scratch//:tokenizer.json)", - "--no-llama3=true", # don't do llama3 template prompt encoding - "--sharding=false", # don't shard this - ], - data = [ - "@TinyLlama-120M-scratch", - "@TinyLlama-120M-scratch//:config.json", - "@TinyLlama-120M-scratch//:model.safetensors", - "@TinyLlama-120M-scratch//:tokenizer.json", - ], - deps = [":llama_lib"], -) - cc_binary( name = "Llama-3.1-8B-Instruct", args = [ @@ -70,7 +53,6 @@ cc_binary( deps = [":llama_lib"], ) - cc_binary( name = "Llama-3.2-1B-Instruct", args = [ @@ -102,7 +84,31 @@ cc_binary( ], deps = [":llama_lib"], ) -# + +cc_binary( + name = "TinyLlama-Stories-15M", + args = [ + "--config=$(location :tinyllama_stories15M_json)", + "--weights=$(location @Karpathy-TinyLlama-Stories15M//file)", + "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", + "--prompt='Once upon a time, there was a little girl named Lily.'", + "--no-llama3=1", # don't do template prompt encoding, I'm a simple model + "--sharding=false", # don't shard me, I'm so small + ], + data = [ + ":tinyllama_stories15M_json", + "@Karpathy-TinyLlama-Stories15M//file", + "@Karpathy-TinyLlama-Tokenizer//file", + ], + deps = [":llama_lib"], +) + +write_file( + name = "tinyllama_stories15M_json", + out = "config.json", + content = ['{"bos_token_id":1,"eos_token_id":2,"hidden_act":"silu","hidden_size":288,"intermediate_size":768,"max_position_embeddings":256,"model_type":"llama","num_attention_heads":6,"num_hidden_layers":6,"num_key_value_heads":6,"rms_norm_eps":1e-05,"hf_rope_impl":false,"rope_scaling":null,"rope_theta":10000.0}'], +) + zig_cc_binary( name = "test-implementation", @@ -117,27 +123,31 @@ zig_cc_binary( ], main = "test.zig", deps = [ - "//third_party/tigerbeetle:flags", "@zml//async", - "@zml//metax", + "@zml//stdx", "@zml//zml", ], ) -zig_cc_binary( +native_test( name = "test_tokenizer", - main = "test_tokenizer.zig", - deps = [ - "//third_party/tigerbeetle:flags", - "@zml//stdx", - "@zml//zml", - ], + src = "@zml//zml/tokenizer:main", # Note: all Llama-3.x tokenizers are the same, # but using the 3.2-1B version because downloading the tokenizer triggers downloading the model. args = [ - "--tokenizer=$(location @Meta-Llama-3.2-1B-Instruct//:tokenizer)", + "--tokenizer=$(location @Meta-Llama-3.2-1B-Instruct//:tokenizer.json)", + """--prompt='Examples of titles: +📉 Stock Market Trends +🍪 Perfect Chocolate Chip Recipe +Evolution of Music Streaming +Remote Work Productivity Tips +Artificial Intelligence in Healthcare +🎮 Video Game Development Insights +'""", + # this correspond to encoding with HF tokenizers, with bos=False + "--expected=41481,315,15671,512,9468,241,231,12937,8152,50730,198,9468,235,103,24118,39520,32013,26371,198,35212,3294,315,10948,45910,198,25732,5664,5761,1968,26788,198,9470,16895,22107,304,39435,198,9468,236,106,8519,4140,11050,73137,198", ], - data = ["@Meta-Llama-3.2-1B-Instruct//:tokenizer"], + data = ["@Meta-Llama-3.2-1B-Instruct//:tokenizer.json"], ) mtree_spec( diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 6f6594f2..a162b432 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -26,6 +26,7 @@ pub const LlamaLM = struct { rope_theta: f32, max_position_embeddings: usize, rms_norm_eps: f32, + hf_rope_impl: bool = true, }; pub const Options = struct { @@ -47,7 +48,7 @@ pub const LlamaLM = struct { self.model.num_heads = @intCast(config.num_attention_heads); self.model.num_kv_heads = @intCast(config.num_key_value_heads); self.model.rope_opts = .{ - .impl = .sequential, + .impl = if (config.hf_rope_impl) .sequential else .interleaved, .freq_base = config.rope_theta, }; for (self.model.layers) |*layer| { diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 3cf2670a..7bf19b7b 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -27,9 +27,9 @@ pub fn tokenizePromptLlama3(allocator: std.mem.Allocator, tokenizer: zml.tokeniz var encoder = try tokenizer.encoder(); defer encoder.deinit(); - const start_header_id = tokenizer.token_to_id("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header_id = tokenizer.token_to_id("<|end_header_id|>") orelse return error.NoSuchToken; - const eot_id = tokenizer.token_to_id("<|eot_id|>") orelse return error.NoSuchToken; + const start_header_id = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header_id = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const eot_id = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; const newline_id = (try encoder.encode("\n"))[0]; try tokens.append(config.bos_token_id); @@ -130,9 +130,11 @@ pub fn generateText( tracer.frameEnd(frame_id, try std.fmt.bufPrintZ(tracer_buffer, "Generated token {}/{}", .{ i + 1, output_tokens_len })); - // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + // extract the generated token from the buffer, async + var fut_tok_buf = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + _ = try fut_tok_buf.awaitt(); const generated_token = generated_token_buffer[0]; + // de-tokenize generated token into a string const chunk = try tokenizer_decoder.next(@intCast(generated_token)) orelse unreachable; num_tokens_generated = i; @@ -168,7 +170,7 @@ const params = clap.parseParamsComptime( \\--seed random seed (optional) \\--seq-len sequence length \\--create-options platform creation options JSON, defaults to {} - \\--no-llama3 skip prompt template + \\--no-llama3 skip prompt template \\--sharding default: true: sharding on or off ); @@ -312,7 +314,7 @@ pub fn asyncMain() !void { var timer = try stdx.time.Timer.start(); defer log.info("Loaded tokenizer from {s} [{}]", .{ tok, timer.read() }); - break :blk try zml.tokenizer.Tokenizer.from_file(model_arena.allocator(), tok); + break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), tok); } else { log.err("Missing --tokenizer", .{}); return; diff --git a/examples/llama/test_tokenizer.zig b/examples/llama/test_tokenizer.zig deleted file mode 100644 index 304a6ccc..00000000 --- a/examples/llama/test_tokenizer.zig +++ /dev/null @@ -1,51 +0,0 @@ -const std = @import("std"); -const log = std.log.scoped(.@"//llama:test_tokenizer"); - -const flags = @import("tigerbeetle/flags"); -const zml = @import("zml"); - -const Flags = struct { - tokenizer: []const u8, - prompt: []const u8 = - \\Examples of titles: - \\📉 Stock Market Trends - \\🍪 Perfect Chocolate Chip Recipe - \\Evolution of Music Streaming - \\Remote Work Productivity Tips - \\Artificial Intelligence in Healthcare - \\🎮 Video Game Development Insights - \\ - , - expected: []const u8 = "128000,41481,315,15671,512,9468,241,231,12937,8152,50730,198,9468,235,103,24118,39520,32013,26371,198,35212,3294,315,10948,45910,198,25732,5664,5761,1968,26788,198,9470,16895,22107,304,39435,198,9468,236,106,8519,4140,11050,73137,198", - verbose: bool = false, -}; - -pub fn main() !void { - var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; - const allocator = gpa.allocator(); - - var raw_args = std.process.args(); - const args = flags.parse(&raw_args, Flags); - - log.info("\tLoading tokenizer from {s}", .{args.tokenizer}); - var tokenizer = try zml.aio.detectFormatAndLoadTokenizer(allocator, args.tokenizer); - log.info("✅\tLoaded tokenizer from {s}", .{args.tokenizer}); - defer tokenizer.deinit(); - - const prompt_tok = try tokenizer.encode(allocator, args.prompt, .{ .debug = args.verbose }); - - log.info("Input: {s}\nOutput: {d}", .{ args.prompt, prompt_tok }); - if (args.expected.len > 0) { - var expected = try std.ArrayList(u32).initCapacity(allocator, args.prompt.len); - var it = std.mem.splitSequence(u8, args.expected, ","); - while (it.next()) |int_token| { - const tok = try std.fmt.parseInt(u32, int_token, 10); - try expected.append(tok); - } - if (std.mem.eql(u32, expected.items, prompt_tok)) { - log.info("All good !", .{}); - } else { - log.err("Doesn't match expected: {d}", .{expected.items}); - } - } -} diff --git a/examples/third_party/tigerbeetle/BUILD.bazel b/examples/third_party/tigerbeetle/BUILD.bazel deleted file mode 100644 index 23baa5b4..00000000 --- a/examples/third_party/tigerbeetle/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -# Files in this folder come from the very fine folks of TigerBeetle -# and is licensed under Apache 2.0 attribution license. -# https://github.com/tigerbeetle/tigerbeetle -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "flags", - import_name = "tigerbeetle/flags", - main = "flags.zig", - visibility = ["//visibility:public"], -) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9cfa5fb5..651a1a55 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -732,7 +732,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 3bc89430..b1701259 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -4,6 +4,7 @@ zig_library( name = "stdx", srcs = [ "debug.zig", + "flags.zig", "io.zig", "json.zig", "math.zig", diff --git a/examples/third_party/tigerbeetle/flags.zig b/stdx/flags.zig similarity index 86% rename from examples/third_party/tigerbeetle/flags.zig rename to stdx/flags.zig index f4bc8d40..c672398a 100644 --- a/examples/third_party/tigerbeetle/flags.zig +++ b/stdx/flags.zig @@ -84,6 +84,13 @@ pub fn parse(args: *std.process.ArgIterator, comptime CliArgs: type) CliArgs { }; } +/// Parse CLI arguments for current process. +/// See `stdx.flags.parse` documentation for more. +pub fn parseProcessArgs(comptime CliArgs: type) CliArgs { + var args = std.process.args(); + return parse(&args, CliArgs); +} + fn parse_commands(args: *std.process.ArgIterator, comptime Commands: type) Commands { comptime assert(@typeInfo(Commands) == .Union); comptime assert(std.meta.fields(Commands).len >= 2); @@ -573,89 +580,3 @@ pub fn default_value(comptime field: std.builtin.Type.StructField) ?field.type { else null; } - -// CLI parsing makes a liberal use of `fatal`, so testing it within the process is impossible. We -// test it out of process by: -// - using Zig compiler to build this very file as an executable in a temporary directory, -// - running the following main with various args and capturing stdout, stderr, and the exit code. -// - asserting that the captured values are correct. -pub const main = if (@import("root") != @This()) - // For production builds, don't include the main function. - // This is `if __name__ == "__main__":` at comptime! -{} else struct { - const CliArgs = union(enum) { - empty, - prefix: struct { - foo: u8 = 0, - foo_bar: u8 = 0, - opt: bool = false, - option: bool = false, - }, - pos: struct { flag: bool = false, positional: struct { - p1: []const u8, - p2: []const u8, - p3: ?u32 = null, - p4: ?u32 = null, - } }, - required: struct { - foo: u8, - bar: u8, - }, - values: struct { - int: u32 = 0, - size: ByteSize = .{ .value = 0 }, - boolean: bool = false, - path: []const u8 = "not-set", - optional: ?[]const u8 = null, - choice: enum { marlowe, shakespeare } = .marlowe, - }, - - pub const help = - \\ flags-test-program [flags] - \\ - ; - }; - - pub fn main() !void { - var gpa_allocator = std.heap.GeneralPurposeAllocator(.{}){}; - const gpa = gpa_allocator.allocator(); - - var args = try std.process.argsWithAllocator(gpa); - defer args.deinit(); - - const cli_args = parse(&args, CliArgs); - - const stdout = std.io.getStdOut(); - const out_stream = stdout.writer(); - switch (cli_args) { - .empty => try out_stream.print("empty\n", .{}), - .prefix => |values| { - try out_stream.print("foo: {}\n", .{values.foo}); - try out_stream.print("foo-bar: {}\n", .{values.foo_bar}); - try out_stream.print("opt: {}\n", .{values.opt}); - try out_stream.print("option: {}\n", .{values.option}); - }, - .pos => |values| { - try out_stream.print("p1: {s}\n", .{values.positional.p1}); - try out_stream.print("p2: {s}\n", .{values.positional.p2}); - try out_stream.print("p3: {?}\n", .{values.positional.p3}); - try out_stream.print("p4: {?}\n", .{values.positional.p4}); - try out_stream.print("flag: {}\n", .{values.flag}); - }, - .required => |required| { - try out_stream.print("foo: {}\n", .{required.foo}); - try out_stream.print("bar: {}\n", .{required.bar}); - }, - .values => |values| { - try out_stream.print("int: {}\n", .{values.int}); - try out_stream.print("size: {}\n", .{values.size.bytes()}); - try out_stream.print("boolean: {}\n", .{values.boolean}); - try out_stream.print("path: {s}\n", .{values.path}); - try out_stream.print("optional: {?s}\n", .{values.optional}); - try out_stream.print("choice: {?s}\n", .{@tagName(values.choice)}); - }, - } - } -}.main; - -// Note: I removed tests to not pull too many deps diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 7b032605..2cbc84a5 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -1,4 +1,5 @@ pub const debug = @import("debug.zig"); +pub const flags = @import("flags.zig"); pub const io = @import("io.zig"); pub const json = @import("json.zig"); pub const math = @import("math.zig"); diff --git a/zml/aio.zig b/zml/aio.zig index 55155c48..26d3e73c 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -10,6 +10,7 @@ const posix = @import("posix.zig"); pub const gguf = @import("aio/gguf.zig"); pub const nemo = @import("aio/nemo.zig"); pub const safetensors = @import("aio/safetensors.zig"); +pub const tinyllama = @import("aio/tinyllama.zig"); pub const torch = @import("aio/torch.zig"); pub const yaml = @import("aio/yaml.zig"); @@ -35,6 +36,8 @@ pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) try gguf.open(allocator, model_path) else if (std.mem.endsWith(u8, model_path, ".pt")) try torch.open(allocator, model_path) + else if (std.mem.endsWith(u8, model_path, ".tinyllama")) + try tinyllama.open(allocator, model_path) else { std.debug.panic("File extension not recognized: {s}", .{model_path}); }; @@ -590,6 +593,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; obj.* = try zml.Buffer.from(platform, buf_with_metadata); + _ = try obj.awaitt(); } else { return error.BufferNotFound; }; diff --git a/zml/aio/tinyllama.zig b/zml/aio/tinyllama.zig index e3fe3f9a..8114c90f 100644 --- a/zml/aio/tinyllama.zig +++ b/zml/aio/tinyllama.zig @@ -129,35 +129,3 @@ fn splitBuff(store: *zml.aio.BufferStore, comptime fmt: []const u8, sh: anytype, } return off; } - -pub fn loadTokenizer(allocator: std.mem.Allocator, tokenizer_path: []const u8, vocab_size: u32) !zml.tokenizer.Tokenizer { - const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); - defer tokenizer_file.close(); - var tok_reader = std.io.bufferedReader(tokenizer_file.reader()); - const r = tok_reader.reader(); - - const max_token_len = try r.readInt(u32, .little); - const special_tokens: zml.tokenizer.Tokenizer.SpecialTokens = .{ - .unk = 0, - .bos = 1, - .eos = 2, - }; - var tokenizer = try zml.tokenizer.Tokenizer.init(allocator, vocab_size, max_token_len, null, special_tokens, true); - var i: u32 = 0; - while (readToken(&tokenizer, &r)) : (i += 1) { - // Pass - } else |_| { - if (i < vocab_size) { - zml.log.info("Read {d} words out of {?d}", .{ i, vocab_size }); - } - tokenizer.vocab_size = i; - } - try tokenizer.rewriteByteFallbackTokens(); - return tokenizer; -} - -fn readToken(tokenizer: *zml.tokenizer.Tokenizer, tok_reader: anytype) !void { - const score: f32 = @bitCast(try tok_reader.readInt(u32, .little)); - const len: usize = @intCast(try tok_reader.readInt(u32, .little)); - try tokenizer.readTokenInto(score, len, tok_reader); -} diff --git a/zml/buffer.zig b/zml/buffer.zig index a9fba67a..b4a06a91 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -27,12 +27,29 @@ const log = std.log.scoped(.zml); /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { + pub const Shard = struct { + api: *const pjrt.Api, + buffer: *pjrt.Buffer, + ready_event: ?*pjrt.Event = null, + ready: bool = false, + + pub fn awaitt(self: *Shard) !void { + if (self.ready) { + return; + } + if (self.ready_event orelse self.buffer.getReadyEvent(self.api)) |ev| { + try ev.awaitt(self.api); + } + self.ready = true; + } + }; + _shape: Shape, _api: *const pjrt.Api, _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; - pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const Shards = std.BoundedArray(Shard, MAX_NUM_SHARDS); /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { @@ -56,44 +73,49 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - .{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + }); + res._shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + .ready_event = event, }); - - frames.appendAssumeCapacity(frame); } - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); - res._shards.appendAssumeCapacity(pjrt_buffer); - } return res; } + pub fn awaitt(self: *Buffer) !*Buffer { + for (self._shards.slice()) |*shard| { + try shard.awaitt(); + } + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); + for (pjrt_buffers) |pjrt_buffer| { + shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + }); + } return .{ ._api = platform.pjrt_api, ._shape = shape_, @@ -108,6 +130,11 @@ pub const Buffer = struct { return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); } + pub fn fromSlice2(platform: Platform, dimz: anytype, s: anytype) !Buffer { + const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); + return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); + } + /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { @@ -115,6 +142,11 @@ pub const Buffer = struct { return try from(platform, host_buffer); } + pub fn fromBytes(platform: Platform, shape_: Shape, bytes: []const u8) !Buffer { + const host_buffer = HostBuffer.fromBytes(shape_, bytes); + return try from(platform, host_buffer); + } + /// Creates a Buffer with a single element. pub fn scalar(platform: Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); @@ -168,7 +200,9 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, .data = &bytes }; - return try from(platform, host_buffer); + var buf = try from(platform, host_buffer); + _ = try buf.awaitt(); + return buf; } test constant { @@ -233,9 +267,9 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.awaitt(self._api); } return res; } @@ -245,22 +279,23 @@ pub const Buffer = struct { /// The returned `HostBuffer` doesn't own the memory. pub fn toHost(self: Buffer, output: []u8) !HostBuffer { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, output); + var hb = HostBuffer.fromBytes(self.shape(), output); + hb._event = maybe_event; + hb._api = self._api; + hb._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return hb; } /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); + var output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); - if (maybe_event) |event| { - try event.await_(self._api); - } + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, @constCast(output.data)); + output._event = maybe_event; + output._api = self._api; + output._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; return output; } @@ -268,8 +303,8 @@ pub const Buffer = struct { /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *const Buffer) void { - for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._api); + for (self._shards.constSlice()) |shard| { + shard.buffer.deinit(self._api); } } diff --git a/zml/exe.zig b/zml/exe.zig index b34d5e9b..b053f8bf 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -210,11 +210,11 @@ pub const BaseExe = struct { .non_donatable_input_indices = &.{}, }) catch unreachable; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn serialize(self: BaseExe, writer: anytype) !void { @@ -316,7 +316,7 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { const model_sharding = ctx.buffers.len; stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {}-sharded tensor into a {}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); for (buffer._shards.constSlice(), 0..) |shard, d| { - ctx.buffers[d][ctx.index] = shard; + ctx.buffers[d][ctx.index] = shard.buffer; } ctx.index += 1; } @@ -344,7 +344,7 @@ fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Bu ctx.index += 1; if (i >= ctx.buffer_shapes.len) return; - var shards: Buffer.Shards = .{}; + var shards: std.BoundedArray(*pjrt.Buffer, Buffer.MAX_NUM_SHARDS) = .{}; for (ctx.buffers) |buff| { shards.appendAssumeCapacity(buff[i]); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index cd7e3a60..da96cff9 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -1,6 +1,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); + const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -22,6 +24,9 @@ pub const HostBuffer = struct { managed: u5, unmanaged, } = .unmanaged, + _ready: bool = true, + _api: ?*const pjrt.Api = null, + _event: ?*pjrt.Event = null, /// Allocates a HostBuffer with the given shape. /// The memory is left undefined. @@ -255,6 +260,21 @@ pub const HostBuffer = struct { }; } + pub fn awaitt(self: *HostBuffer) !*HostBuffer { + if (self._ready) { + return self; + } + if (self._event) |ev| { + if (self._api) |api| { + try ev.awaitt(api); + } else { + stdx.debug.panic("Got an event but no API. This should not happen.", .{}); + } + } + self._ready = true; + return self; + } + pub fn format( self: HostBuffer, comptime fmt: []const u8, diff --git a/zml/module.zig b/zml/module.zig index 0c1bc19f..a003896d 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -905,8 +905,8 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables setFlag(&options, "xla_gpu_enable_triton_gemm", false); setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); - setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); - setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); + // setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); + // setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 7408e074..cf68dba0 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -17,7 +17,6 @@ const log = std.log.scoped(.zml); pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; -pub const Buffer = pjrt.Buffer; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; @@ -63,13 +62,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return buffer; + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -130,6 +125,62 @@ pub const Client = opaque { } }; +pub const Buffer = opaque { + pub const inner = InnerMixin(pjrt.Buffer).inner; + + pub fn deinit(self: *Buffer, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { + return try self.inner().getDevice(api); + } + + pub fn delete(self: *Buffer, api: *const Api) void { + self.inner().delete(api); + } + + pub fn isDeleted(self: *const Buffer, api: *const Api) bool { + return self.inner().isDeleted(api); + } + + pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { + return self.inner().isOnCpu(api); + } + + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { + return @ptrCast(try self.inner().toHostBuffer(api, dst)); + } + + pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { + return self.inner().getElementType(api); + } + + pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { + return self.inner().getDimensions(api); + } + + pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { + return try self.inner().getUnpaddedDimensions(api); + } + + pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { + return try self.inner().getOnDeviceSizeInBytes(api); + } + + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { + return @ptrCast(self.inner().copyToDevice(api, device)); + } + + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { + return @ptrCast(self.inner().getReadyEvent(api)); + } + + pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { + return try self.inner().getOpaqueDeviceMemoryDataPointer(api); + } +}; + pub const Event = opaque { pub const inner = InnerMixin(pjrt.Event).inner; @@ -145,7 +196,8 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) !void { + pub const await_ = awaitt; + pub fn awaitt(self: *Event, api: *const Api) !void { defer self.deinit(api); if (self.isReady(api)) { @@ -201,13 +253,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { diff --git a/zml/tensor.zig b/zml/tensor.zig index 23ab003b..f2e37497 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2419,6 +2419,10 @@ pub const Tensor = struct { /// of the operator which is backend specific. update_fn: *const fn (Tensor, Tensor) Tensor = increment, + pub fn maximum(old_value: Tensor, new_value: Tensor) Tensor { + return old_value.maximum(new_value.convert(old_value.dtype())); + } + pub fn increment(old_value: Tensor, new_value: Tensor) Tensor { return old_value.add(new_value.convert(old_value.dtype())); } diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 2e0a71f1..3bd9da98 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -16,6 +16,7 @@ zig_library( name = "tokenizer", import_name = "zml/tokenizer", main = "tokenizer.zig", + srcs = ["homemade.zig"], visibility = ["//visibility:public"], deps = [ "//async", @@ -30,6 +31,8 @@ zig_cc_binary( main = "main.zig", visibility = ["//visibility:public"], deps = [ + "//stdx", + "//async", ":tokenizer", ], ) diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig index d5c252fa..538a25a1 100644 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ b/zml/tokenizer/hftokenizers/hftokenizers.zig @@ -91,7 +91,7 @@ pub const Decoder = struct { }; pub const HFTokenizer = opaque { - pub fn from_file(model: []const u8) !*HFTokenizer { + pub fn fromFile(model: []const u8) !*HFTokenizer { return @ptrCast(c.hftokenizers_new(ffi.ZigSlice.from(model))); } @@ -107,7 +107,7 @@ pub const HFTokenizer = opaque { return Decoder.init(self); } - pub fn token_to_id(self: *HFTokenizer, token: []const u8) ?u32 { + pub fn tokenToId(self: *HFTokenizer, token: []const u8) ?u32 { return c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); } }; diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig new file mode 100644 index 00000000..448dd0e4 --- /dev/null +++ b/zml/tokenizer/homemade.zig @@ -0,0 +1,1215 @@ +//! Text tokenizer implementations +//! Disclaimer this is not a very robust implementation: +//! In particular the normalization is pretty minimalist, only works with ascii, and don't do unicode normalization. +//! Mostly used for testing models that don't have an official HF/sentencepiece tokenizer. +const builtin = @import("builtin"); +const std = @import("std"); + +const testing = std.testing; + +const log = std.log.scoped(.@"zml/tokenizer"); + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Normalizer); + std.testing.refAllDecls(Tokenizer); +} + +/// Byte Pair Encoding tokenizer generally used for LLM. +pub const Tokenizer = struct { + tokens: [][]const u8, + token_lookup: std.StringHashMapUnmanaged(u32), + special_tokens: SpecialTokens, + + scores: []f32, + max_token_len: u32, + normalizer: ?Normalizer, + // Allows to split unknown unicode characters into bytes. + byte_fallback: bool = false, + + arena_state: std.heap.ArenaAllocator, + vocab_size: u32, + next_token_id: u32 = 0, + + pub const SpecialTokens = struct { + eos: u32, + bos: u32, + unk: u32, + pad: u32 = std.math.maxInt(u32), + hard_space: u32 = std.math.maxInt(u32), + }; + + pub fn init( + allocator: std.mem.Allocator, + vocab_size: u32, + max_token_len: u32, + normalizer: ?Normalizer, + special_tokens: SpecialTokens, + alloc_tokens: bool, + ) !Tokenizer { + var arena_state = std.heap.ArenaAllocator.init(allocator); + errdefer arena_state.deinit(); + const arena = arena_state.allocator(); + + var token_lookup: std.StringHashMapUnmanaged(u32) = .{}; + errdefer token_lookup.deinit(arena); + + try token_lookup.ensureTotalCapacity(arena, @intCast(vocab_size)); + + const tokens: [][]const u8 = if (alloc_tokens) try arena.alloc([]u8, vocab_size) else &.{}; + errdefer if (alloc_tokens) arena.free(tokens); + + const scores: []f32 = if (alloc_tokens) try arena.alloc(f32, vocab_size) else &.{}; + errdefer if (alloc_tokens) arena.free(scores); + + return .{ + .tokens = tokens, + .scores = scores, + .max_token_len = max_token_len, + .token_lookup = token_lookup, + .arena_state = arena_state, + .normalizer = normalizer, + .vocab_size = vocab_size, + .special_tokens = special_tokens, + }; + } + + pub fn deinit(self: Tokenizer) void { + self.arena_state.deinit(); + } + + pub fn encoder(self: *Tokenizer) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *Tokenizer) !Decoder { + return Decoder.init(self); + } + + /// Reads a new word directly into the tokenizer arena. + pub fn readTokenInto(self: *Tokenizer, score: f32, len: usize, tok_reader: anytype) !void { + const arena = self.arena_state.allocator(); + + const token = try arena.alloc(u8, len); + const n = try tok_reader.read(token); + std.debug.assert(n == len); + + return self.addOwnedToken(score, token); + } + + /// Adds a new token (and copy it) + pub fn addToken(self: *Tokenizer, score: f32, token: []const u8) !void { + const arena = self.arena_state.allocator(); + return self.addOwnedToken(score, try arena.dupe(u8, token)); + } + + /// Adds a new token (without copying it) + pub fn addOwnedToken(self: *Tokenizer, score: f32, token: []const u8) void { + const i = self.next_token_id; + std.debug.assert(i < self.vocab_size); + self.next_token_id += 1; + + self.scores[i] = score; + self.tokens[i] = token; + const v = self.token_lookup.getOrPutAssumeCapacity(token); + if (!v.found_existing) { + v.value_ptr.* = i; + } + } + + pub fn addOwnedTokenByIndex(self: *Tokenizer, i: u32, score: f32, token: []const u8) void { + std.debug.assert(i < self.vocab_size); + self.next_token_id += 1; + self.scores[i] = score; + self.tokens[i] = token; + const v = self.token_lookup.getOrPutAssumeCapacity(token); + if (!v.found_existing) { + v.value_ptr.* = @intCast(i); + } + } + + pub fn lookup(self: *const Tokenizer, str: []const u8) ?u32 { + return self.token_lookup.get(str); + } + + pub fn tokenToId(self: *const Tokenizer, token: []const u8) ?u32 { + return self.token_lookup.get(token); + } + + pub const EncodeOptions = struct { + /// Should the beginning of sentence '' token be added. + add_bos: bool = true, + add_eos: bool = false, + pad_to: u32 = 0, + // Print tokenization intermediary steps. + debug: bool = false, + }; + + pub fn encode(self: *const Tokenizer, allocator: std.mem.Allocator, raw: []const u8, options: EncodeOptions) ![]u32 { + if (options.debug) log.debug("Tokenizer.encode('{s}')", .{raw}); + const input = if (self.normalizer) |n| try n.normalize(allocator, raw) else raw; + defer if (self.normalizer) |_| allocator.free(input); + if (options.debug) log.debug("Tokenizer.encode.normalize -> '{s}'", .{input}); + + // Allocate a buffer that can fit all indices as well as extra character if requested. + // We then slice it so that the token merging code doesn't see the bos token. + const tok_buff_alloc = try allocator.alloc(u32, @max(options.pad_to, input.len + 2)); + const tok_buff = if (options.add_bos) tok_buff_alloc[1..] else tok_buff_alloc; + + const MergeState = union(enum) { ready: u32, nope, hard_space, idk }; + const mergeable = try allocator.alloc(MergeState, tok_buff.len); + + var num_tokens: usize = 0; + var it: CharTokenIterator = .{ .input = input }; + while (try it.nextCodepointToken(self)) |token| : (num_tokens += 1) { + tok_buff[num_tokens] = token; + mergeable[num_tokens] = if (token == self.special_tokens.hard_space) + .hard_space + else + .idk; + } + + var stable_prefix: usize = 0; + var stable_off: usize = 0; + while (true) { + // This code is a bit overcomplicated cause I'm abstracting over two algorithms: + // BPE and sentencepiece unigram model. + // Normally BPE is pre-split on spaces then the regular merge algorithm is applied. + // With unigram model you work at sentence level and you handle spaces as you would any other bytes, + // hoping the final tokens mostly align with spaces. + // This seemed like a good idea, but is kinda bad because I had to add special code to speed up BPE + // by detecting when the first "word" is treated and can be safely removed from sequence. + // Also it doesn't work well with BPE vocab which have multi-space tokens (for indentation) + // and have custom splitting rules. + // This is fine for now cause we now have bindings to HF tokenizers for complexe use cases + // and are only using this for tinyllama/gguf models. + // If we come back to use this in production, the implementation would gain in speed/clarity + // by splitting in two. + // The merging token logic isn't that complicated anyway. + + // Step by step visualization of the progress. + if (options.debug) { + var _debug_buf: [256]u8 = undefined; + var _debug_alloc = std.heap.FixedBufferAllocator.init(&_debug_buf); + var debug_progress = std.ArrayList(u8).init(_debug_alloc.allocator()); + self.decodeWithOpts(&debug_progress, tok_buff[0..num_tokens], .{ .sep = "|" }) catch {}; + log.debug("tokens: {d} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.items }); + } + var best_score: f32 = -1e10; + var best_token: u32 = 0; + var best_idx: ?usize = null; + var input_off: usize = stable_off; + + // Find best tokens to merge in all available tokens + for (stable_prefix..num_tokens - 1) |i| { + if (tok_buff[i] == self.special_tokens.unk) { + input_off += 1; + continue; + } + const cur_tok = self.tokens[tok_buff[i]]; + defer input_off += cur_tok.len; + + // Lookup merge for current token, if not already done. + switch (mergeable[i]) { + .nope => continue, + .ready => {}, + .hard_space => { + // Since tokens are not allowed to merge through hard sep, + // we don't need to merge the sentence-wide best token. + // We can just merge the best token since beginning. + if (best_idx != null) break; + // OTOH if there was no merge possible since beginning, + // we can skip the beginning in future iterations. + stable_prefix = i + 1; + stable_off = input_off + cur_tok.len; + continue; + }, + .idk => { + + // Special tokens can't be concatenated. + if (builtin.mode == .Debug and tok_buff[i] != self.special_tokens.unk) { + // Detects memory corruption of tokens. + if (cur_tok.len == 0 or cur_tok.len > self.max_token_len) @panic("Token looks corrupted !"); + + if (!std.mem.eql(u8, cur_tok, input[input_off..][0..cur_tok.len])) { + log.err("current token '{s}' not found in input string '{s}' !", .{ cur_tok, input[input_off..] }); + @panic("invalid tokenization"); + } + } + const next_tok = self.tokens[tok_buff[i + 1]]; + // if `next_tok` is `.unk`, length is 1; otherwise, it's the length of the token. + const next_tok_len = if (tok_buff[i + 1] == self.special_tokens.unk) 1 else next_tok.len; + const concat_tokens = input[input_off..][0 .. cur_tok.len + next_tok_len]; + // Save the result + mergeable[i] = if (self.lookup(concat_tokens)) |tok| + .{ .ready = tok } + else + .nope; + }, + } + + switch (mergeable[i]) { + .idk, .hard_space => unreachable, + .nope => continue, + .ready => |tok| { + if (self.scores[tok] > best_score) { + best_score = self.scores[tok]; + best_token = tok; + best_idx = i; + } + }, + } + } + + if (best_idx) |bidx| { + // Apply the merge. + tok_buff[bidx] = best_token; + std.mem.copyForwards(u32, tok_buff[bidx + 1 ..], tok_buff[bidx + 2 .. num_tokens]); + std.mem.copyForwards(MergeState, mergeable[bidx + 1 ..], mergeable[bidx + 2 .. num_tokens]); + num_tokens -= 1; + // We got two new merge lookups to do. + mergeable[bidx] = .idk; + if (bidx > 0 and mergeable[bidx - 1] != .hard_space) mergeable[bidx - 1] = .idk; + } else { + // No merge candidate => we are done ! + break; + } + } + + if (options.add_eos) { + tok_buff[num_tokens] = self.special_tokens.eos; + num_tokens += 1; + } + if (options.add_bos) { + tok_buff_alloc[0] = self.special_tokens.bos; + num_tokens += 1; + } + if (num_tokens < options.pad_to) { + for (num_tokens..options.pad_to) |i| { + tok_buff_alloc[i] = self.special_tokens.pad; + } + num_tokens = options.pad_to; + } + + // Release extra memory we don't need anymore. + allocator.free(mergeable); + _ = allocator.resize(tok_buff_alloc, num_tokens); + return tok_buff_alloc[0..num_tokens]; + } + + /// Returns a slice corresponding to the given id. Handles unknown ids and special ids. + pub fn lookupPiece(self: *const Tokenizer, id: usize) []const u8 { + return if (id == self.special_tokens.bos or id == self.special_tokens.eos or id == self.special_tokens.pad) + "" + else if (id == self.special_tokens.unk) + "" + else if (id > self.tokens.len) + "" // this means we received an invalid id, but we didn't want to panic. + else + self.tokens[id]; + } + + /// Converts the given slice of tokens back into bytes. + /// Note that if the tokenizer allows sub-unicode bytes, it's possible + /// the output is not valid utf8. + pub fn decode(self: *const Tokenizer, allocator: std.mem.Allocator, input: []const u32) error{OutOfMemory}![]u8 { + var output = std.ArrayList(u8).init(allocator); + errdefer output.deinit(); + + try self.decodeWithOpts(&output, input, .{}); + return output.toOwnedSlice(); + } + + pub fn decodeWithOpts( + self: *const Tokenizer, + output: *std.ArrayList(u8), + input: []const u32, + opts: struct { sep: []const u8 = "" }, + ) error{OutOfMemory}!void { + const escaped = if (self.normalizer) |n| n.escapedSpace() else null; + // Flag used to indicate if the first dummy whitespace has been consumed. + for (input) |id| { + // Retrieve the slice corresponding to the id. + var piece = self.lookupPiece(id); + + // Convert `▁` to a regular space. + if (escaped) |escspc| { + // we modify piece inside the loop, so we can use it in the condition + while (std.mem.startsWith(u8, piece, escspc)) { + piece = piece[escspc.len..]; + // don't output a space at beginning of text. + if (output.items.len > 0) try output.append(' '); + } + } + + try output.appendSlice(piece); + if (opts.sep.len > 0) try output.appendSlice(opts.sep); + } + } + + /// Some tokenizers have bytes encoded in hex like this: "<0x40>". + /// This break the tokenization algorithm because the input text + /// will contain "@" not "<0x40>", + /// and if the input contains "<0x40>" it needs to not be treated as a single byte. + /// So we replace byte fallbacks strings, by their corresponding character. + /// This enables the normal tokenization algorithm to work. + pub fn rewriteByteFallbackTokens(tokenizer: *Tokenizer) !void { + tokenizer.byte_fallback = true; + var single_bytes = try tokenizer.arena_state.allocator().alloc(u8, 256); + var byte_fallback_buf = "<0x00>".*; + + for (0..256) |i| { + const c: u8 = @truncate(i); + single_bytes[i] = c; + + // First lookup the byte fallback entry. + // Note: we assume upper case, but we could try both upper and lower case if needed. + _ = std.fmt.bufPrintIntToSlice(byte_fallback_buf[3..5], c, 16, .upper, .{ .fill = '0', .width = 2 }); + const entry = tokenizer.token_lookup.getEntry(&byte_fallback_buf) orelse { + log.err("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback token {s}", .{byte_fallback_buf}); + return error.InvalidInput; + }; + + // Check if the character is already present in the vocab. + // In that case, nothing to do, + // but note that the fallback token will be "unreachable", + // ie there is no way the tokenizer can produce it. + if (tokenizer.token_lookup.get(&.{c})) |_| continue; + + const idx: u32 = entry.value_ptr.*; + tokenizer.token_lookup.removeByPtr(entry.key_ptr); + tokenizer.addOwnedTokenByIndex(idx, tokenizer.scores[idx], single_bytes[i .. i + 1]); + } + } +}; + +test Tokenizer { + const allocator = std.testing.allocator; + const special_tokens: Tokenizer.SpecialTokens = .{ + .unk = 0, + .bos = 1, + .eos = 2, + }; + + var tokenizer = try Tokenizer.init(allocator, 10, 5, null, special_tokens, true); + defer tokenizer.deinit(); + + try tokenizer.addToken(10, "hello"); + try tokenizer.addToken(3.5, "world"); + + try testing.expect(tokenizer.lookup("hello") == 0); + try testing.expect(tokenizer.lookup("world") == 1); + + // TODO: test Tokenizer.decode, Tokenizer.encode, Tokenizer.readTokenInto +} + +pub const Encoder = struct { + inner: *Tokenizer, + arena: std.heap.ArenaAllocator, + current_ids: []const u32 = &.{}, + + fn init(inner: *Tokenizer) !Encoder { + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. + _ = try arena.allocator().alloc(u32, 4096); + std.debug.assert(arena.reset(.retain_capacity)); + return .{ .inner = inner, .arena = arena }; + } + + pub fn reset(self: *Encoder) void { + self.current_ids = &.{}; + std.debug.assert(self.arena.reset(.retain_capacity)); + } + + pub fn deinit(self: *Encoder) void { + self.arena.deinit(); + } + + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + self.reset(); + const res = try self.inner.encode(self.arena.allocator(), input, .{ + .add_bos = true, + .add_eos = false, + .pad_to = 0, + // Print tokenization intermediary steps. + .debug = false, + }); + self.current_ids = res; + return res; + } + + pub fn ids(self: *const Encoder) []const u32 { + return self.current_ids; + } +}; + +pub const Decoder = struct { + const StringBuffer = std.BoundedArray(u8, 128); + const TokensIdsBuffer = std.BoundedArray(u32, 4); + + inner: *Tokenizer, + arena: std.heap.ArenaAllocator, + + current_string: ?[]const u8 = null, + last_string: StringBuffer = .{ .len = 0 }, + last_token_ids: TokensIdsBuffer = .{ .len = 0 }, + + fn init(inner: *Tokenizer) !Decoder { + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. + _ = try arena.allocator().alloc(u32, 4096); + std.debug.assert(arena.reset(.retain_capacity)); + return .{ .inner = inner, .arena = arena }; + } + + pub fn deinit(self: *Decoder) void { + self.arena.deinit(); + } + + pub fn reset(self: *Decoder) void { + std.debug.assert(self.arena.reset(.retain_capacity)); + self.current_string = null; + } + + pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { + self.reset(); + const res = try self.inner.decode(self.arena.allocator(), ids); + self.current_string = res; + return res; + } + + pub fn string(self: *const Decoder) []const u8 { + return self.current_string; + } + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + if (self.last_token_ids.len >= self.last_token_ids.capacity()) { + _ = self.last_token_ids.orderedRemove(0); + } + self.last_token_ids.appendAssumeCapacity(token_id); + const new_string = try self.decode(self.last_token_ids.constSlice()); + if (self.last_string.len == 0) { + self.last_string = try StringBuffer.fromSlice(new_string); + return new_string; + } + var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); + var it = view.iterator(); + while (it.nextCodepointSlice()) |cp| { + const start = it.i - cp.len; + if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { + const chunk = new_string[self.last_string.len - start ..]; + self.last_string = try StringBuffer.fromSlice(new_string); + return chunk; + } + } + return null; + } +}; + +/// Given a slice, split it in the most simple tokens using the given tokenizer tokens. +/// The output of this can be used to initialize the tokenization algorithm. +/// Normally we split the input text into utf8 codepoint, +/// but if we find an unknown codepoint we either split it in bytes, or use the special "unknown" token, +/// depending on the tokenizer configuration. +const CharTokenIterator = struct { + state: union(enum) { by_codepoint, by_byte: u8 } = .by_codepoint, + input: []const u8, + + fn nextCodepointToken(self: *CharTokenIterator, tokenizer: *const Tokenizer) error{ TruncatedInput, Utf8InvalidStartByte }!?u32 { + if (self.input.len == 0) return null; + return switch (self.state) { + .by_byte => |*byte_left| { + const idx = tokenizer.lookup(self.input[0..1]) orelse { + // Normally this has been caught when calling `rewriteByteFallbackTokens`. + std.debug.panic("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback for token '<0x{X:02}>'", .{self.input[0]}); + }; + + self.input = self.input[1..]; + byte_left.* -|= 1; + if (byte_left.* == 0) self.state = .by_codepoint; + return idx; + }, + .by_codepoint => { + // Try to lookup valid utf8 codepoint first. + const utf8_len = try std.unicode.utf8ByteSequenceLength(self.input[0]); + if (self.input.len < utf8_len) return error.TruncatedInput; + if (tokenizer.lookup(self.input[0..utf8_len])) |idx| { + self.input = self.input[utf8_len..]; + return idx; + } + + // Otherwise split in bytes if it's allowed. + if (tokenizer.byte_fallback) { + // TODO: replace this by a continue statement next time we bump Zig. + self.state = .{ .by_byte = utf8_len }; + return self.nextCodepointToken(tokenizer); + } + + // Or mark the full utf8 codepoint as unknown. + log.debug("Token not found for char '{s}'", .{self.input[0..utf8_len]}); + self.input = self.input[utf8_len..]; + return tokenizer.special_tokens.unk; + }, + }; + } +}; + +test CharTokenIterator { + const special_tokens: Tokenizer.SpecialTokens = .{ .unk = 0, .bos = 1, .eos = 2 }; + var tokenizer = try Tokenizer.init(std.testing.allocator, 16, 4, null, special_tokens, true); + defer tokenizer.deinit(); + + tokenizer.addOwnedToken(1.0, ""); // 0 + tokenizer.addOwnedToken(1.0, ""); // 1 + tokenizer.addOwnedToken(1.0, ""); // 2 + tokenizer.addOwnedToken(1.0, "ζ"); // 3 + tokenizer.addOwnedToken(1.0, &.{0xE2}); // 4: ℳ, first byte + tokenizer.addOwnedToken(1.0, &.{0x84}); // 5: ℳ, second byte + tokenizer.addOwnedToken(1.0, &.{0xB3}); // 6: ℳ, third byte + tokenizer.addOwnedToken(1.0, "L"); // 7 + + // No byte fallback + { + tokenizer.byte_fallback = false; + var it: CharTokenIterator = .{ .input = "ζℳL" }; + var res: std.BoundedArray(u32, 8) = .{}; + while (try it.nextCodepointToken(&tokenizer)) |token| { + res.appendAssumeCapacity(token); + } + try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 0, 7 }, res.constSlice()); + } + + // with byte fallback + { + tokenizer.byte_fallback = true; + var it: CharTokenIterator = .{ .input = "ζℳL" }; + var res: std.BoundedArray(u32, 8) = .{}; + while (try it.nextCodepointToken(&tokenizer)) |token| { + res.appendAssumeCapacity(token); + } + try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 4, 5, 6, 7 }, res.constSlice()); + } +} + +/// Text normalizer. +/// Most tokenizer assumes the input text have been prepocessed with on of those. +pub const Normalizer = struct { + /// Space token used by sentencepiece derived tokenizer. + pub const sentencepiece_space = "▁"; // \xe2\x96\x81 + + _whitespace: std.BoundedArray(u8, 8) = .{}, + + flags: packed struct { + remove_extra_whitespaces: bool, + add_dummy_prefix: bool, + add_dummy_suffix: bool, + /// Cheap lower casing. + /// TODO: try to match Python "lower" + lower_case_ascii: bool, + /// cheap ascii punct splitting. + // doing this processing ahead of time simplifies the logic + split_on_punct_ascii: bool, + }, + + pub fn init(flags: std.meta.FieldType(Normalizer, .flags), escaped_whitespace: ?[]const u8) Normalizer { + var res: Normalizer = .{ .flags = flags }; + if (escaped_whitespace) |escaped| { + res._whitespace.appendSliceAssumeCapacity(escaped); + } + return res; + } + + pub inline fn escapedSpace(self: Normalizer) ?[]const u8 { + return if (self._whitespace.len > 1) self._whitespace.constSlice() else null; + } + + fn addSlice(data: []const u8, consumed: usize, normalized: *std.ArrayList(u8), normalized_to_origin: *std.ArrayList(usize)) !void { + try normalized.appendSlice(data); + for (data) |_| try normalized_to_origin.append(consumed); + } + + pub const Result = struct { + /// Normalized string + normalized: []const u8, + /// Mapping between chars in the original string and chars in the new string + normalized_to_origin: []const usize, + + pub fn deinit(self: Result, allocator: std.mem.Allocator) void { + allocator.free(self.normalized); + allocator.free(self.normalized_to_origin); + } + }; + + /// Simplifed version of Sentencepiece normalizer. + /// + /// Llama2 uses a normalizer called "identity" so this basically only handles trailing + /// whitespaces and replaces whitespace with the "▁" (U+2581) character. + pub fn normalize(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) ![]const u8 { + const res = try self.normalizeWithMapping(allocator, input); + allocator.free(res.normalized_to_origin); + return res.normalized; + } + + /// Returns both the normalized string and a mapping between the normalized string and the original. + pub fn normalizeWithMapping(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) !Result { + // Number of bytes consumed from the input. + var consumed: usize = 0; + var trimmed_input = input; + + // Skip leading whitespaces. + if (self.flags.remove_extra_whitespaces) { + while (trimmed_input.len != 0) { + if (trimmed_input[0] != ' ') break; + trimmed_input = trimmed_input[1..]; + consumed += 1; + } + } + + // If the trimmed input is empty, we are done. + if (trimmed_input.len == 0) { + return .{ .normalized = &.{}, .normalized_to_origin = &.{} }; + } + + // Pre-allocate outputs + const space = self.escapedSpace() orelse " "; + const overhead = if (self.flags.split_on_punct_ascii) space.len + 1 else space.len; + var normalized = try std.ArrayList(u8).initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); + errdefer normalized.deinit(); + var normalized_to_origin = try std.ArrayList(usize).initCapacity(allocator, normalized.capacity); + errdefer normalized_to_origin.deinit(); + + // If the spec asks for it, add a whitespace at the beginning. + if (self.flags.add_dummy_prefix) try addSlice(space, consumed, &normalized, &normalized_to_origin); + + var is_prev_space: bool = true; + var is_prev_word: bool = false; + + while (trimmed_input.len != 0) { + // NOTE(Corendos): This might feel weird but normally the slice we get comes from a normalizing process and can contain multiple codepoints. + // Since we have an "identity" normalizer, each slice is actually a unicode character. + const multibyte_length = try std.unicode.utf8ByteSequenceLength(trimmed_input[0]); + var slice = trimmed_input[0..multibyte_length]; + const origin = consumed; + consumed += multibyte_length; + trimmed_input = trimmed_input[multibyte_length..]; + + if (self.flags.remove_extra_whitespaces and is_prev_space) { + while (slice.len > 0 and slice[0] == ' ') { + slice = slice[1..]; + } + if (slice.len == 0) continue; + } + is_prev_space = slice[slice.len - 1] == ' '; + + if (slice.len == 1) ascii: { + // The more advanced logic only works with ascii atm + var byte = slice[0]; + if (self.escapedSpace() != null and byte == ' ') { + // replace the space token by the special token + try addSlice(space, origin, &normalized, &normalized_to_origin); + is_prev_word = false; + break :ascii; + } else if (self.flags.split_on_punct_ascii) { + if (is_prev_word and isPunct(slice)) { + // Insert a space, but continue handling the rest + try addSlice(space, origin, &normalized, &normalized_to_origin); + } + } + if (self.flags.lower_case_ascii) { + byte = std.ascii.toLower(byte); + } + try normalized.append(byte); + try normalized_to_origin.append(origin); + } else { + // we can safely copy to the output. + try addSlice(slice, origin, &normalized, &normalized_to_origin); + } + is_prev_word = !is_prev_space and !isPunct(slice); + } + + // Skip trailing whitespaces + if (self.flags.remove_extra_whitespaces) { + while (std.mem.endsWith(u8, normalized.items, space)) { + const length = normalized.items.len - space.len; + consumed = normalized_to_origin.items[length]; + try normalized.resize(length); + try normalized_to_origin.resize(length); + } + } + + try normalized_to_origin.append(consumed); + + std.debug.assert(normalized_to_origin.items.len == normalized.items.len + 1); + + if (self.flags.add_dummy_suffix) try addSlice(space, consumed, &normalized, &normalized_to_origin); + + return .{ + .normalized = try normalized.toOwnedSlice(), + .normalized_to_origin = try normalized_to_origin.toOwnedSlice(), + }; + } + + pub fn wellKnown(impl: KnownImplementation) Normalizer { + return switch (impl) { + .sentencepiece => init(.{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, sentencepiece_space), + .llama3 => init(.{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, null), + .gpt2 => init(.{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, null), + }; + } + + pub fn fromHfJson(config: std.json.ObjectMap) error{InvalidNormalizerJson}!Normalizer { + var normalizer: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = false, + .add_dummy_suffix = false, + .add_dummy_prefix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + } }; + + // Normalizer config can be a single normalizer, or a sequence of normalizers. + const maybe_steps = objectGet(config, .array, "normalizers"); + const steps = if (maybe_steps) |st| st.items else &.{std.json.Value{ .object = config }}; + + for (steps) |step_val| { + if (step_val != .object) { + return error.InvalidNormalizerJson; + } + const step = step_val.object; + + const step_type = objectGet(step, .string, "type") orelse { + return error.InvalidNormalizerJson; + }; + if (std.mem.eql(u8, "Prepend", step_type)) { + normalizer.flags.add_dummy_prefix = true; + } else if (std.mem.eql(u8, "Append", step_type)) { + normalizer.flags.add_dummy_suffix = true; + } else if (std.mem.eql(u8, "Replace", step_type)) { + const pattern = objectGet(step, .object, "pattern") orelse return error.InvalidNormalizerJson; + const str_pattern = objectGet(pattern, .string, "String") orelse return error.InvalidNormalizerJson; + + if (std.mem.eql(u8, str_pattern, " ")) { + normalizer._whitespace.appendSliceAssumeCapacity( + objectGet(step, .string, "content") orelse return error.InvalidNormalizerJson, + ); + } else { + log.warn("Normalizer Replace pattern not supported: '{s}' -> '{s}'", .{ str_pattern, objectGet(pattern, .string, "content") orelse "" }); + } + } else { + log.warn("Unknown normalizer type: {s}", .{step_type}); + } + } + + return normalizer; + } + + test "Normalizer.fromHfJson" { + const config_json = + \\{ + \\ "type": "Sequence", + \\ "normalizers": [ + \\ { + \\ "type": "Prepend", + \\ "prepend": "▁" + \\ }, + \\ { + \\ "type": "Replace", + \\ "pattern": { + \\ "String": " " + \\ }, + \\ "content": "▁" + \\ } + \\ ] + \\} + ; + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const config = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), config_json, .{}); + const normalizer = try Normalizer.fromHfJson(config.object); + + const expected = Normalizer{ + ._whitespace = .{ .buffer = [_]u8{ 0xe2, 0x96, 0x81 } ++ [_]u8{0} ** 5, .len = 3 }, + .flags = .{ + .remove_extra_whitespaces = false, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, + }; + try std.testing.expectEqual(expected.flags, normalizer.flags); + try std.testing.expectEqualStrings(expected.escapedSpace().?, normalizer.escapedSpace().?); + } +}; +pub const KnownImplementation = enum(u8) { + sentencepiece, + gpt2, + llama3, +}; + +fn isPunct(unicode_char: []const u8) bool { + // TODO use unicode categories + if (unicode_char.len > 1) return false; + + return switch (unicode_char[0]) { + ' ', '\t' => false, + 0...8 => true, + 10...31 => true, + '!'...'/' => true, + ':'...'@' => true, + '['...'`' => true, + '{'...'~' => true, + else => false, + }; +} + +test Normalizer { + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + } }; + const res = try n.normalizeWithMapping(testing.allocator, "Hellŏ world!"); + defer res.deinit(testing.allocator); + + try testing.expectEqualSlices(u8, " Hellŏ world!", res.normalized); + try testing.expectEqualSlices( + usize, + // H e l l ŏ ␣ w o r l d ! + &.{ 0, 0, 1, 2, 3, 4, 4, 6, 8, 9, 10, 11, 12, 13, 14 }, + res.normalized_to_origin, + ); + } + + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = true, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + } }; + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, " Hello world! ", res); + } + + { + const n = Normalizer.init( + .{ + .remove_extra_whitespaces = false, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, + Normalizer.sentencepiece_space, + ); + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, "▁Hello▁▁world!", res); + } + + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = true, + .lower_case_ascii = true, + .split_on_punct_ascii = false, + } }; + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, "hello world! ", res); + } + + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = true, + .lower_case_ascii = false, + .split_on_punct_ascii = true, + } }; + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, "Hello world ! ", res); + } +} + +/// gpt2 had their own way of storing text. +/// Unfortunately this has contaminated other models. +/// This implementation precompupte a mapping between bytes encoded with GPT2 algorithm, +/// into utf8 bytes, and do lookups at runtime. +pub const Gpt2TextDecoder = struct { + const Code = std.BoundedArray(u8, 2); + + // TODO: benchmark this is more efficient than doing the conversion at runtime. + code_to_byte: std.AutoArrayHashMap(Code, u8), + + pub fn init(allocator: std.mem.Allocator) !Gpt2TextDecoder { + var self = Gpt2TextDecoder{ + .code_to_byte = std.AutoArrayHashMap(Code, u8).init(allocator), + }; + try self.code_to_byte.ensureTotalCapacity(256); + errdefer unreachable; + + var n: usize = 0; + for (0..256) |index| { + var code: Code = .{ .buffer = .{ 0, 0 }, .len = 0 }; // 0-init + const i: u8 = @intCast(index); + if (isPrintableByte(i)) { + if (std.ascii.isASCII(i)) { + code.appendAssumeCapacity(i); + } else { + const codepoint: u21 = @as(u21, @intCast(i)); + code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); + } + } else { + const codepoint: u21 = 256 + @as(u21, @intCast(n)); + code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); + n += 1; + } + + self.code_to_byte.putAssumeCapacityNoClobber(code, i); + } + return self; + } + + pub fn deinit(self: *Gpt2TextDecoder) void { + self.code_to_byte.deinit(); + } + + /// Transform bytes representing text under the gpt2 encoding, + /// and write to the `unicode` buffer utf-8 bytes. + pub fn decode(self: Gpt2TextDecoder, unicode: *std.ArrayList(u8), bytes: []const u8) ![]const u8 { + const start = unicode.items.len; + var it = std.unicode.Utf8Iterator{ .i = 0, .bytes = bytes }; + while (it.nextCodepointSlice()) |codepoint| { + const code: Code = switch (codepoint.len) { + 1 => .{ .buffer = .{ codepoint[0], 0 }, .len = 1 }, // 0-init + 2 => .{ .buffer = .{ codepoint[0], codepoint[1] }, .len = 2 }, + else => return error.InvalidInput, + }; + const byte = self.code_to_byte.get(code) orelse return error.InvalidInput; + try unicode.append(byte); + } + return unicode.items[start..]; + } + + inline fn isPrintableByte(c: u8) bool { + return ('!' <= c and c <= '~') or (0xa1 <= c and c <= 0xac) or (0xae <= c and c <= 0xff); + } +}; + +test Gpt2TextDecoder { + var decoder = try Gpt2TextDecoder.init(testing.allocator); + defer decoder.deinit(); + + var out = std.ArrayList(u8).init(testing.allocator); + defer out.deinit(); + + // Ascii is not changed. + try testing.expectEqualStrings("getTitle", try decoder.decode(&out, "getTitle")); + // Leading space are represented with 'Ġ' + try testing.expectEqualStrings(" UINavigationController", try decoder.decode(&out, "ĠUINavigationController")); + // Russian is wild + try testing.expectEqualStrings(" работ", try decoder.decode(&out, "ĠÑĢабоÑĤ")); +} + +/// Open a json file in HF format and load the vocab from it. +pub fn fromHfJson(allocator: std.mem.Allocator, tokenizer_path: []const u8) !Tokenizer { + const file = try std.fs.cwd().openFile(tokenizer_path, .{}); + defer file.close(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const file_content = try file.readToEndAlloc(arena, 32 * 1024 * 1024); + + const info = try std.json.parseFromSliceLeaky(std.json.Value, arena, file_content, .{ + .duplicate_field_behavior = .use_last, + }); + const main_object = switch (info) { + .object => |obj| if (obj.get("added_tokens") == null or obj.get("model") == null) { + return error.InvalidFormat; + } else obj, + else => return error.InvalidFormat, + }; + + const model = objectGet(main_object, .object, "model") orelse return error.InvalidFormat; + const vocab = objectGet(model, .object, "vocab") orelse return error.InvalidFormat; + const added_tokens = if (objectGet(main_object, .array, "added_tokens")) |added| added.items else &.{}; + const vocab_size: u32 = @intCast(vocab.count() + added_tokens.len); + + const normalizer = if (objectGet(main_object, .object, "normalizer")) |normalizer_config| + try Normalizer.fromHfJson(normalizer_config) + else + Normalizer.wellKnown(.llama3); + + // delay init of special tokens. + var tokenizer = try Tokenizer.init(allocator, vocab_size, 256, normalizer, undefined, true); + errdefer tokenizer.deinit(); + + // Buffer containing all concatenated tokens. + // Reserve a big chunk, to avoid grow event, but release over-allocated memory. + var all_tokens = try std.ArrayList(u8).initCapacity(tokenizer.arena_state.allocator(), file_content.len); + const original_alloc = all_tokens.items.ptr; + // A re-alloc event here means we have invalidated all slices inside the tokenizer. + // If this is too annoying we could switch to a custom type instead of slices. + defer { + std.debug.assert(all_tokens.items.ptr == original_alloc); + } + + // gpt2 based tokenizer got a special way of encoding unicode. + // we don't know in advance if this will be used by this tokenizer or not. + // so we assume it is the case, but if we find some unicode character, + // outside of the range used by gpt2 we know it was wrong, and start over. + var is_gpt2_vocab: bool = true; + var gpt2_decoder = try Gpt2TextDecoder.init(allocator); + defer gpt2_decoder.deinit(); + var it = vocab.iterator(); + while (it.next()) |kv| { + const token = gpt2_decoder.decode(&all_tokens, kv.key_ptr.*) catch |err| { + switch (err) { + error.InvalidInput => { + is_gpt2_vocab = false; + break; + }, + else => return err, + } + }; + const idx: u32 = @intCast(kv.value_ptr.*.integer); + tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); + } + + if (!is_gpt2_vocab) { + // We where wrong, this is not a gpt2 vocab, start over, + // and reset the tokenizer state. + tokenizer.next_token_id = 0; + tokenizer.token_lookup.clearRetainingCapacity(); + all_tokens.clearRetainingCapacity(); + it = vocab.iterator(); + while (it.next()) |kv| { + const idx: u32 = @intCast(kv.value_ptr.*.integer); + const token = try dup(&all_tokens, kv.key_ptr.*); + tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); + } + } + + // More tokens, typically added during fine tuning of the model. + for (added_tokens) |token_obj| { + if (token_obj != .object) return error.InvalidFormat; + const v = objectGet(token_obj.object, .string, "content") orelse return error.InvalidFormat; + const id: u32 = @intCast(objectGet(token_obj.object, .integer, "id") orelse return error.InvalidFormat); + const token = try if (is_gpt2_vocab) + gpt2_decoder.decode(&all_tokens, v) + else + dup(&all_tokens, v); + + tokenizer.addOwnedTokenByIndex(id, 0, token); + } + // We won't add more tokens here, let release. + all_tokens.shrinkAndFree(all_tokens.items.len); + + var unk = tokenizer.lookup(""); + if (objectGet(model, .integer, "unk_token")) |unk_tok| { + unk = @intCast(unk_tok); + } + + tokenizer.special_tokens = .{ + // TODO allow users to specify special tokens or read them from a tokenizer_config.json file + .bos = tokenizer.lookup("") orelse tokenizer.lookup("<|begin_of_text|>") orelse @panic("bos token not found !"), + .eos = tokenizer.lookup("") orelse tokenizer.lookup("<|end_of_text|>") orelse @panic("eos token not found !"), + .unk = unk orelse std.math.maxInt(u32), + }; + + const byte_fallback = objectGet(model, .bool, "byte_fallback") orelse false; + if (!byte_fallback and unk == null) { + // GPT2 tokenizer have byte fallback already encoded in the model, + // but the json generally don't have the field set. + // We can detect it though because they don't specify an unknown token. + if (is_gpt2_vocab) { + tokenizer.byte_fallback = true; + } else { + log.warn("The given tokenizer can't handle unknown token: no unknown token was set, and byte_fallback is disabled too ! The tokenizer will panic when facing unknown tokens.", .{}); + } + } else if (byte_fallback) { + try tokenizer.rewriteByteFallbackTokens(); + } + return tokenizer; +} + +/// Returns a copy of the given string, stored inside the given ArrayList. +fn dup(buffer: *std.ArrayList(u8), str: []const u8) ![]const u8 { + const n = buffer.items.len; + try buffer.appendSlice(str); + return buffer.items[n..]; +} + +/// Returns the given entry in a json object only if it has the right type. +fn objectGet( + object: std.json.ObjectMap, + comptime kind: std.meta.FieldEnum(std.json.Value), + key: []const u8, +) ?std.meta.FieldType(std.json.Value, kind) { + const val = object.get(key) orelse return null; + if (val != kind) return null; + return @field(val, @tagName(kind)); +} + +pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, tokenizer_path: []const u8, vocab_size: u32) !Tokenizer { + const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); + defer tokenizer_file.close(); + var tok_reader = std.io.bufferedReader(tokenizer_file.reader()); + const r = tok_reader.reader(); + + const max_token_len = try r.readInt(u32, .little); + const special_tokens: Tokenizer.SpecialTokens = .{ + .unk = 0, + .bos = 1, + .eos = 2, + }; + var tokenizer = try Tokenizer.init(allocator, vocab_size, max_token_len, null, special_tokens, true); + var i: u32 = 0; + while (readToken(&tokenizer, &r)) : (i += 1) { + // Pass + } else |_| { + if (i < vocab_size) { + log.info("Read {d} words out of {?d}", .{ i, vocab_size }); + } + tokenizer.vocab_size = i; + } + try tokenizer.rewriteByteFallbackTokens(); + return tokenizer; +} + +fn readToken(tokenizer: *Tokenizer, tok_reader: anytype) !void { + const score: f32 = @bitCast(try tok_reader.readInt(u32, .little)); + const len: usize = @intCast(try tok_reader.readInt(u32, .little)); + try tokenizer.readTokenInto(score, len, tok_reader); +} diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index b8efd01e..d3554f1d 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -1,22 +1,65 @@ const std = @import("std"); -const tokenizer = @import("zml/tokenizer"); +const log = std.log.scoped(.@"//zml/tokenizer"); + +const asynk = @import("async"); +const stdx = @import("stdx"); +const zml_tokenizer = @import("zml/tokenizer"); + +const Flags = struct { + tokenizer: []const u8, + prompt: []const u8, + expected: []const u8 = "", + verbose: bool = false, +}; pub fn main() !void { - const model2 = "/private/var/tmp/_bazel_steeve/a67b810d44f2a673ebbd5bab86ccd5cc/external/zml~~huggingface~Meta-Llama-3.1-8B-Instruct/tokenizer.json"; + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} - var sp = try tokenizer.Tokenizer.from_file(std.heap.c_allocator, model2); - defer sp.deinit(); +pub fn asyncMain() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; + const allocator = gpa.allocator(); - std.debug.print("Loaded model\n", .{}); + const args = stdx.flags.parseProcessArgs(Flags); - var encoder = try sp.encoder(); + log.info("\tLoading tokenizer from {s}", .{args.tokenizer}); + var tokenizer = try zml_tokenizer.Tokenizer.fromFile(allocator, args.tokenizer); + log.info("✅\tLoaded tokenizer from {s}", .{args.tokenizer}); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); defer encoder.deinit(); - var decoder = try sp.decoder(); + var decoder = try tokenizer.decoder(); defer decoder.deinit(); - const ids = try encoder.encode("Hello, world! plane pouet plane"); - const decoded = try decoder.decode(ids); + const prompt_tok = try encoder.encode(args.prompt); - std.debug.print("{d}\n{s}\n", .{ ids, decoded }); + log.info("Input: {s}\nOutput: {d}", .{ args.prompt, prompt_tok }); + + var errors: u8 = 0; + { + const reconstructed = try decoder.decode(prompt_tok); + if (!std.mem.eql(u8, args.prompt, reconstructed)) { + log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed}); + errors += 1; + } + } + + if (args.expected.len > 0) { + var expected = try std.ArrayList(u32).initCapacity(allocator, args.prompt.len); + var it = std.mem.splitSequence(u8, args.expected, ","); + while (it.next()) |int_token| { + const tok = try std.fmt.parseInt(u32, int_token, 10); + try expected.append(tok); + } + if (!std.mem.eql(u32, expected.items, prompt_tok)) { + log.err("Doesn't match expected: {d}", .{expected.items}); + errors += 1; + } + } + + if (errors == 0) log.info("All good !", .{}); + + std.process.exit(errors); } diff --git a/zml/tokenizer/sentencepiece/sentencepiece.zig b/zml/tokenizer/sentencepiece/sentencepiece.zig index a5ce8d47..7f4edc69 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.zig +++ b/zml/tokenizer/sentencepiece/sentencepiece.zig @@ -164,7 +164,7 @@ pub const Decoder = struct { }; pub const SentencePieceProcessor = opaque { - pub fn from_file(model: []const u8) !*SentencePieceProcessor { + pub fn fromFile(model: []const u8) !*SentencePieceProcessor { const sp: *SentencePieceProcessor = @ptrCast(c.SentencePieceProcessor_new()); errdefer sp.deinit(); try assertOk(c.SentencePieceProcessor_Load(@ptrCast(sp), ffi.ZigSlice.from(model))); @@ -183,7 +183,7 @@ pub const SentencePieceProcessor = opaque { return try Decoder.init(self); } - pub fn token_to_id(self: *SentencePieceProcessor, token: []const u8) u32 { + pub fn tokenToId(self: *SentencePieceProcessor, token: []const u8) u32 { return @intCast(c.SentencePieceProcessor_PieceToId(@ptrCast(self), ffi.ZigSlice.from(token))); } }; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 10307db7..a28b3a06 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -3,15 +3,19 @@ const hftokenizers = @import("hftokenizers"); const sentencepiece = @import("sentencepiece"); const asynk = @import("async"); +const homemade = @import("homemade.zig"); + const Tokenizers = enum { hftokenizers, sentencepiece, + homemade, }; pub const Tokenizer = union(Tokenizers) { pub const Encoder = union(Tokenizers) { hftokenizers: hftokenizers.Encoder, sentencepiece: sentencepiece.Encoder, + homemade: homemade.Encoder, pub fn deinit(self: *Encoder) void { switch (self.*) { @@ -41,6 +45,7 @@ pub const Tokenizer = union(Tokenizers) { pub const Decoder = union(Tokenizers) { hftokenizers: hftokenizers.Decoder, sentencepiece: sentencepiece.Decoder, + homemade: homemade.Decoder, pub fn deinit(self: *Decoder) void { switch (self.*) { @@ -81,14 +86,22 @@ pub const Tokenizer = union(Tokenizers) { hftokenizers: *hftokenizers.HFTokenizer, sentencepiece: *sentencepiece.SentencePieceProcessor, + homemade: *homemade.Tokenizer, - pub fn from_file(_: std.mem.Allocator, model: []const u8) !Tokenizer { + pub fn fromFile(allocator: std.mem.Allocator, model: []const u8) !Tokenizer { if (std.mem.endsWith(u8, model, ".pb")) { - return .{ .sentencepiece = try asynk.callBlocking(sentencepiece.SentencePieceProcessor.from_file, .{model}) }; + return .{ .sentencepiece = try asynk.callBlocking(sentencepiece.SentencePieceProcessor.fromFile, .{model}) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try asynk.callBlocking(hftokenizers.HFTokenizer.from_file, .{model}) }; + return .{ .hftokenizers = try asynk.callBlocking(hftokenizers.HFTokenizer.fromFile, .{model}) }; } + + if (std.mem.endsWith(u8, model, ".tinyllama")) { + const tokenizer = try allocator.create(homemade.Tokenizer); + tokenizer.* = try asynk.callBlocking(homemade.fromTinyLlamaFile, .{ allocator, model, 32000 }); + return .{ .homemade = tokenizer }; + } + return error.InvalidArgument; } @@ -110,9 +123,9 @@ pub const Tokenizer = union(Tokenizers) { }; } - pub fn token_to_id(self: Tokenizer, token: []const u8) ?u32 { + pub fn tokenToId(self: Tokenizer, token: []const u8) ?u32 { return switch (self) { - inline else => |v| v.token_to_id(token), + inline else => |v| v.tokenToId(token), }; } };