diff --git a/MODULE.bazel b/MODULE.bazel index 73468e4f..4c04352a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -21,7 +21,7 @@ bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") bazel_dep(name = "rules_rust", version = "0.63.0") -bazel_dep(name = "rules_zig", version = "0.12.2") +bazel_dep(name = "rules_zig", version = "0.12.3") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7a3758f1..6c24ff42 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -113,8 +113,8 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.12/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.14/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.8/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", @@ -176,7 +176,7 @@ "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_zig/0.12.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_zig/0.12.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_zig/0.12.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -333,9 +333,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", - "https://bcr.bazel.build/modules/rules_cc/0.2.11/MODULE.bazel": "e94f24f065bf2191dba2dace951814378b66a94bb3bcc48077492fe0508059b5", "https://bcr.bazel.build/modules/rules_cc/0.2.12/MODULE.bazel": "4216c383ce3223c7dfbd3afcb09c7056265156150b2a3bd3fd9d02949a7ee1bc", - "https://bcr.bazel.build/modules/rules_cc/0.2.12/source.json": "593d822a2ce7b70d5fad3195f7eb64d519dde94d7c20b0b79e6c23891a79b2bd", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/source.json": "55d0a4587c5592fad350f6e698530f4faf0e7dd15e69d43f8d87e220c78bea54", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", @@ -409,8 +409,8 @@ "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", "https://bcr.bazel.build/modules/rules_zig/0.12.1/MODULE.bazel": "5953094b681c212eebe3dd4275809c52b9177aeaa058d8f1bff433822c0dcd01", - "https://bcr.bazel.build/modules/rules_zig/0.12.2/MODULE.bazel": "09d2ca5486c5c3bbcaa61057958d35f9b4e1e981e08a5a28ad516bab87e8c4ef", - "https://bcr.bazel.build/modules/rules_zig/0.12.2/source.json": "4cef85ce1b625967ce9c063b8716920e8e7067c38321981a7bfd581ad588f4ba", + "https://bcr.bazel.build/modules/rules_zig/0.12.3/MODULE.bazel": "d2917ed1d6d0510553101be587bd2d2b236a63ac20612c8f26a7fcfb2fb5c47f", + "https://bcr.bazel.build/modules/rules_zig/0.12.3/source.json": "9b599ada6d04fbe0144b8e4092bdc5540d49b8c62a3a13988d6ab1f03ba69ae9", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -1169,7 +1169,7 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "ZaON2eA6LKrwcPU8xDGi0W3+6wD1r6kjb0pSYFC4Se8=", + "bzlTransitiveDigest": "zqRaSoEgRpMcDeWk1K16DqBxqQUILfmS+/URBwsw7CQ=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", @@ -2822,11 +2822,11 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "87SI2jgNGLvFKH9GAVUH7PVF1DJqfUvlXiSe9uiKbnk=", + "bzlTransitiveDigest": "EwflR/Okh5RJpz+iRzxWRcrX2K05f4LfBD8/pK+iE68=", + "usagesDigest": "Cw/jB/LASTlSFqF+jd3/TEs+kI2riEthW9Qh0bSKIoI=", "recordedFileInputs": { "@@//bazel/zig_index.json": "e6c6b29fa227fa9b1806dc03af2b6dab00f52f34d39878557074e1626027259f", - "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" + "@@rules_zig+//zig/private/versions.json": "12c90f64135026b686ef4299859baf0d9b285ef67eddcb21f2190333b915df4d" }, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 635eb087..1f204e77 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -8,7 +8,7 @@ zig_binary( name = "llama", srcs = [ "llama.zig", - ], + ] + glob(["isaac/*.zig", "isaac/token_sampler/*.zig"]), main = "main.zig", visibility = ["//visibility:public"], deps = [ @@ -17,6 +17,10 @@ zig_binary( "//zml", "@com_github_hejsil_clap//:clap", ], + zigopts = select({ + "@platforms//cpu:x86_64": ["-mcpu=native"], + "//conditions:default": [], + }) + ["-freference-trace=16"], ) # TODO(steeve): Re-enable with a small, tokenless huggingface model. diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 606598fb..bad6d546 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -128,13 +128,20 @@ pub const LlamaLM = struct { tokens_: Tensor, token_index: Tensor, kv_cache: KvCache, - rng: Tensor.Rng, - ) struct { Tensor, KvCache, Tensor.Rng } { + ) struct { Tensor, KvCache } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); - const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + + var out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); + + out = out.withPartialTags(.{ .s, .d }); + var logits = if (self.lm_head) |lm_head| + zml.call(lm_head, .forward, .{out}) + else + self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); + + if (logits.dim(.s) > 1) logits = logits.slice1d(.s, .{ .start = -1 }).sq; + return .{ logits.transpose(.{ .d, .b }).toMemory(.host_pinned), updated_kv_cache }; } pub fn sampleTokens( diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6d577695..90a19d69 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -28,9 +28,11 @@ const params = clap.parseParamsComptime( \\--hf-model-path path to the directory containing model weights, config and tokenizer \\--seed random seed (optional) \\--seq-len sequence length + \\--topk top k \\--create-options platform creation options JSON, defaults to {} \\--no-llama3 skip prompt template \\--sharding default: true: sharding on or off + \\--bs batch-size ); pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { @@ -68,6 +70,7 @@ pub fn generateText( kv_cache_: zml.Bufferized(llama.KvCache), tokenizer: zml.tokenizer.Tokenizer, allocator: std.mem.Allocator, + b: u32, seed: u128, prompt: []const u8, skip_llama3_encoding: bool, @@ -82,30 +85,53 @@ pub fn generateText( const platform = mod_generate.platform(); const max_seq_len = llama_.model.max_seq_len; + const tracer: zml.tools.Tracer = .init("ai.zml.token_sampler"); + defer tracer.deinit(); + // init RNG and buffers - var rng = try zml.Tensor.Rng.init(platform, seed); - var generated_token_buffer = [_]u32{undefined}; + const generated_token_buffer = try allocator.alignedAlloc(u32, .@"64", b); + + const probs_shape = mod_prefill.inner.result_shapes[0]; + std.debug.assert(probs_shape.eql(mod_generate.inner.result_shapes[0])); + + const TokenSampler = @import("isaac/TokenSampler.zig"); + const strategy: TokenSampler.Strategy = .{ .top_k = 16, .batchsize = b }; + + var sampler: TokenSampler = switch (b) { + 1 => try TokenSampler.single.SingleTopK(.bf16).init(strategy, allocator, @truncate(seed)), + 4 => try TokenSampler.batched.BatchedTopK(4, .bf16).init(strategy, allocator, @splat(@truncate(seed))), + 8 => try TokenSampler.batched.BatchedTopK(8, .bf16).init(strategy, allocator, @splat(@truncate(seed))), + 16 => try TokenSampler.batched.BatchedTopK(16, .bf16).init(strategy, allocator, @splat(@truncate(seed))), + else => @panic("unsupported --batch-size"), + }; + defer sampler.deinit(); var kv_cache = prefill: { // prepare device buffers for the prefill tokens and their positions - const prefill_buffer = try allocator.alloc(u32, max_seq_len); - @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); + const prefill_buffer = try allocator.alloc(u32, b * max_seq_len); + for (0..b) |i| { + @memcpy(prefill_buffer[i * max_seq_len ..][0..prompt_tok.len], prompt_tok); + } - var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); + var prefill_tokens = try zml.Buffer.fromSlice(platform, .{ .b = b, .s = max_seq_len }, prefill_buffer); defer prefill_tokens.deinit(); var prefill_token_pos = try zml.Buffer.scalar(platform, 0, .u32); defer prefill_token_pos.deinit(); - const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); - _ = try prefilled_tokens.toHost(std.mem.sliceAsBytes(prefill_buffer)); - generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; + const probs_d, const kv_cache = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_ }); + defer probs_d.deinit(); + + // works cause we're using toMemory(.host_pinned) in llama.zig + const probs_h = probs_d.asHostBuffer(); + _ = try sampler.writer.writeAll(probs_h.bytes()); + sampler.pick(generated_token_buffer.ptr); break :prefill kv_cache; }; defer zml.aio.unloadBuffers(&kv_cache); // Prepare for token-by-token generation, // start with the token generated based on the full prompt. - var current_token = try zml.Buffer.fromSlice(platform, .{1}, &generated_token_buffer); + var current_token = try zml.Buffer.fromSlice(platform, .{ .b = b, .s = 1 }, generated_token_buffer); defer current_token.deinit(); const output_tokens_len = max_seq_len - prompt_tok.len - 1; @@ -116,7 +142,7 @@ pub fn generateText( generation: for (0..output_tokens_len + 1) |i| { // collect and print generated sequence - num_tokens_generated += 1; + num_tokens_generated += b; const generated_token = generated_token_buffer[0]; if (try tokenizer_decoder.next(generated_token)) |chunk| { try writer.writeAll(chunk); @@ -124,14 +150,14 @@ pub fn generateText( // check for eos if (i == output_tokens_len) break :generation; - switch (config.eos_token_id.value) { - .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, - .ints => |eos_list| { - for (eos_list) |eos| { - if (generated_token == @as(u32, @intCast(eos))) break :generation; - } - }, - } + // switch (config.eos_token_id.value) { + // .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, + // .ints => |eos_list| { + // for (eos_list) |eos| { + // if (generated_token == @as(u32, @intCast(eos))) break :generation; + // } + // }, + // } // current token pos needs to go into a zml.Buffer const token_pos_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; @@ -139,10 +165,20 @@ pub fn generateText( defer token_pos.deinit(); // call to generate the next token - current_token, kv_cache, rng = mod_generate.call(.{ current_token, token_pos, kv_cache, rng }); + const probs_d, kv_cache = mod_generate.call(.{ current_token, token_pos, kv_cache }); + defer probs_d.deinit(); - // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + { + const probs_d_ready = try probs_d.await(); + const f_sample = tracer.frameStart("sample"); + const probs_h = probs_d_ready.asHostBuffer(); + _ = try sampler.writer.writeAll(probs_h.bytes()); + sampler.pick(generated_token_buffer.ptr); + tracer.frameEnd(f_sample, "sample"); + } + + current_token.deinit(); + current_token = try zml.Buffer.fromSlice(platform, .{ .b = b, .s = 1 }, generated_token_buffer); } const end = std.time.microTimestamp(); const duration = stdx.math.divFloat(f64, end - start, std.time.us_per_s); @@ -243,11 +279,13 @@ pub fn asyncMain() !void { defer store.deinit(); // Write metadata from the config file into the LlamaLm struct. + const b: u32 = cli.args.bs orelse 1; const seq_len: u32 = cli.args.@"seq-len" orelse 256; + const llama_options: llama.LlamaLM.Options = .{ .max_seq_len = seq_len, .sampling_strategy = .{ - .topk = 1, + .topk = cli.args.topk orelse 16, .temperature = 1.0, }, }; @@ -260,19 +298,19 @@ pub fn asyncMain() !void { const llama_tensors: llama.LlamaLM = try .init(compiler_arena.allocator(), config, llama_options, store); // Specify shapes of input arguments - const prefill_tokens_shape = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); - const gen_tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); + const prefill_tokens_shape = zml.Shape.init(.{ .b = b, .s = llama_options.max_seq_len }, .u32); + const gen_tokens_shape = zml.Shape.init(.{ .b = b, .s = 1 }, .u32); const token_idx_shape = zml.Shape.init(.{}, .u32); const dtype = llama_tensors.model.embed_tokens.weight.dtype(); const kv_shape = zml.Shape.init(.{ .layer = llama_tensors.model.layers.len, .k = seq_len, + .b = b, .h = config.num_key_value_heads, .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), }, dtype).withSharding(.{.h}); const kv_cache_shape: zml.ShapeOf(llama.KvCache) = llama.KvCache.initShape(kv_shape); - const rng_shape = zml.Tensor.Rng.shape(); // Compile the model twice, one for prefill, one for generation. var start = try std.time.Timer.start(); @@ -282,7 +320,6 @@ pub fn asyncMain() !void { prefill_tokens_shape, token_idx_shape, kv_cache_shape, - rng_shape, }, platform, }); @@ -293,7 +330,6 @@ pub fn asyncMain() !void { gen_tokens_shape, token_idx_shape, kv_cache_shape, - rng_shape, }, platform, }); @@ -338,6 +374,7 @@ pub fn asyncMain() !void { kv_cache, tokenizer, allocator, + b, seed, prompt[0..], skip_llama3_encoding, diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 7c7c1394..9a93a8ec 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -733,6 +733,7 @@ pub const CustomCallOpts = struct { }; call_target_name: [:0]const u8, + kernel_name: []const u8 = &.{}, has_side_effect: bool, backend_config: ?mlir.Attribute, operand_layouts: ?[]const []const usize = null, @@ -768,6 +769,7 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, .{ "backend_config", backend_config }, }); + if (opts.kernel_name.len > 0) attrs.appendAssumeCapacity(.{ "kernel_name", .string(ctx, opts.kernel_name) }); { var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index cad1ce69..8354ad62 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -440,18 +440,18 @@ pub const Client = opaque { } pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { - try api.call(.PJRT_Client_DmaMap, .{ + _ = try api.call(.PJRT_Client_DmaMap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), .size = @intCast(data.len), }); } - pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { - try api.call(.PJRT_Client_DmaUnmap, .{ + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) void { + _ = api.call(.PJRT_Client_DmaUnmap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), - }); + }) catch {}; } pub const CreateBuffersForAsyncHostToDeviceArgs = struct { diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index efe4abb0..089b3b64 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -37,6 +37,7 @@ zig_library( "floats.zig", "helpers.zig", "hostbuffer.zig", + "Io.zig", "meta.zig", "mlirx.zig", "module.zig", diff --git a/zml/Io.zig b/zml/Io.zig new file mode 100644 index 00000000..ae704ab4 --- /dev/null +++ b/zml/Io.zig @@ -0,0 +1,191 @@ +const builtin = @import("builtin"); +const std = @import("std"); + +const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); +const log = std.log.scoped(.@"zml.io"); + +pub const ShardReader = struct { + pub const NUM_SLOTS = 2; + + pjrt_api: *const pjrt.Api, + pjrt_buffer: *const pjrt.Buffer, + total_size: u64, + chunk_size: usize, + + slots: [NUM_SLOTS]?*pjrt.Event, + dma_buffer: []u8, + bytes_requested: u64, + bytes_activated: u64, + next_request_slot: u1, + next_consume_slot: u1, + + is_primed: bool, + interface: std.io.Reader, + + pub fn init(pjrt_api: *const pjrt.Api, pjrt_buffer: *const pjrt.Buffer, dma_buffer: []u8) ShardReader { + const chunk_size = dma_buffer.len / NUM_SLOTS; + std.debug.assert(chunk_size > 0); + + return .{ + .pjrt_api = pjrt_api, + .pjrt_buffer = pjrt_buffer, + .total_size = pjrt_buffer.getOnDeviceSizeInBytes(pjrt_api) catch unreachable, + .chunk_size = chunk_size, + .slots = .{ null, null }, + .dma_buffer = dma_buffer, + .bytes_requested = 0, + .bytes_activated = 0, + .next_request_slot = 0, + .next_consume_slot = 0, + .is_primed = false, + .interface = .{ .vtable = &vtable, .buffer = &.{}, .seek = 0, .end = 0 }, + }; + } + + fn requestNextChunk(self: *ShardReader) std.io.Reader.StreamError!void { + // const trace = tracer.frameStart("ShardReader.requestNextChunk"); + // defer tracer.frameEnd(trace, "ShardReader.requestNextChunk"); + + const slot_to_fill = self.next_request_slot; + std.debug.assert(self.slots[slot_to_fill] == null); + + if (self.bytes_requested >= self.total_size) return; + + const offset_in_dma = slot_to_fill * self.chunk_size; + const chunk_dma_buf = self.dma_buffer[offset_in_dma .. offset_in_dma + self.chunk_size]; + const remaining_on_device = self.total_size - self.bytes_requested; + const transfer_size = @min(remaining_on_device, chunk_dma_buf.len); + const dest_slice = chunk_dma_buf[0..transfer_size]; + const offset: i64 = @intCast(self.bytes_requested); + + log.debug("ShardReader.requestNextChunk: slot={d}, device_offset={d}B, transfer_size={d}B, remaining={d}B", .{ + slot_to_fill, + offset, + transfer_size, + self.total_size - (self.bytes_requested + transfer_size), + }); + + const event = self.pjrt_buffer.copyRawToHost(self.pjrt_api, dest_slice, offset) catch |err| { + log.err("PJRT copyRawToHost failed: {}", .{err}); + return error.ReadFailed; + }; + + if (event) |ev| self.slots[slot_to_fill] = ev; + self.bytes_requested += transfer_size; + self.next_request_slot = 1 - self.next_request_slot; + } + + fn stream(r: *std.io.Reader, w: *std.io.Writer, limit: std.io.Limit) std.io.Reader.StreamError!usize { + // const trace = tracer.frameStart("ShardReader.stream(vtable)"); + // defer tracer.frameEnd(trace, "ShardReader.stream(vtable)"); + + // From Io.Writer.vtable documentation: + // > In addition to, or instead of writing to `w`, + // > the implementation may choose to store data in `buffer`. + // This is what we do since we need to write in DMA memory. + _ = w; + _ = limit; + const self = @as(*ShardReader, @alignCast(@fieldParentPtr("interface", r))); + + std.debug.assert(r.seek == r.end); + + if (!self.is_primed) { + self.is_primed = true; + if (self.total_size == 0) return error.EndOfStream; + log.debug("ShardReader: Priming pipeline for {d} bytes...", .{self.total_size}); + for (0..NUM_SLOTS) |_| try self.requestNextChunk(); + } else { + try self.requestNextChunk(); + } + + const slot_to_consume = self.next_consume_slot; + + if (self.slots[slot_to_consume]) |event| { + // const trace_await = tracer.frameStart("ShardReader.awaitEvent"); + // defer tracer.frameEnd(trace_await, "ShardReader.awaitEvent"); + + event.awaitBlocking(self.pjrt_api) catch |err| { + log.err("Error awaiting event in stream: {}", .{err}); + return error.ReadFailed; + }; + self.slots[slot_to_consume] = null; + } else { + if (self.bytes_activated >= self.total_size) { + return error.EndOfStream; + } else { + log.err("ShardReader stalled: waiting for slot {d} which has no event.", .{slot_to_consume}); + return error.ReadFailed; + } + } + + const offset_in_dma = slot_to_consume * self.chunk_size; + const remaining_total = self.total_size - self.bytes_activated; + const actual_chunk_size = @min(self.chunk_size, remaining_total); + + self.bytes_activated += actual_chunk_size; + + r.buffer = self.dma_buffer[offset_in_dma .. offset_in_dma + actual_chunk_size]; + r.seek = 0; + r.end = r.buffer.len; + + self.next_consume_slot = 1 - self.next_consume_slot; + + return 0; + } + + const vtable: std.io.Reader.VTable = .{ + .stream = stream, + }; +}; + +test "ShardReader: streamRemaining" { + const zml = @import("zml.zig"); + + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + + const tracer: zml.tools.Tracer = .init("ai.zml.io"); + const trace_test = tracer.frameStart("ShardReader.test.streamRemaining"); + defer tracer.frameEnd(trace_test, "ShardReader.test.streamRemaining"); + + const Local = struct { + fn forward() zml.Tensor { + return .arange(.{ .end = 256 * MB }, .u32); + } + }; + + const x_d = try zml.testing.compileAndCall(platform, Local.forward, .{}); + defer x_d.deinit(); + + const dma_buffer = try allocator.alignedAlloc(u8, .fromByteUnits(4 * KB), 1 * MB); + defer allocator.free(dma_buffer); + + if (platform.target != .cpu) platform.pjrt_client.dmaMap(platform.pjrt_api, dma_buffer) catch {}; + defer if (platform.target != .cpu) platform.pjrt_client.dmaUnmap(platform.pjrt_api, dma_buffer); + + var device_reader: ShardReader = .init(platform.pjrt_api, x_d._shards.get(0), dma_buffer); + + const x_h = try allocator.alignedAlloc(u32, .fromByteUnits(16 * KB), x_d.shape().count()); + defer allocator.free(x_h); + var x_h_writer: std.Io.Writer = .fixed(@ptrCast(x_h)); + + const trace_stream = tracer.frameStart("ShardReader.streamRemaining"); + const bytes_read = try device_reader.interface.streamRemaining(&x_h_writer); + tracer.frameEnd(trace_stream, "ShardReader.streamRemaining"); + + std.log.warn("Device: {f}, host: {d}, read: {d}", .{ + x_d, + 256 * MB * @sizeOf(u32), + bytes_read, + }); + try std.testing.expectEqual(x_d.shape().byteSize(), bytes_read); + try std.testing.expectEqual(256 * MB * @sizeOf(u32), bytes_read); + for (x_h, 0..) |actual, expected| { + errdefer log.err("Mismatch at offset {d}, expected {x}, got {x}", .{ expected, expected, actual }); + try std.testing.expectEqual(expected, actual); + } +} + +const KB = 1024; +const MB = 1024 * KB; diff --git a/zml/buffer.zig b/zml/buffer.zig index e76039d2..dc5c15cc 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const Io = @import("Io.zig"); const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -484,6 +485,62 @@ pub const Buffer = struct { } return deleted; } + + pub fn reader(self: Buffer, allocator: std.mem.Allocator, dma_buffer: []u8) !Reader { + return try .init(allocator, self, dma_buffer); + } + + const Reader = struct { + // TODO: optimize a bit, there is a lot of redundant information across shards. + shards: []Io.ShardReader, + active: u8, + interface: std.Io.Reader, + + pub fn init(allocator: std.mem.Allocator, input: Buffer, dma_buffer: []u8) !Reader { + const num_shards = input._shards.len; + std.debug.assert(num_shards > 0); + const dma_bytes_per_shard = std.math.divCeil(usize, dma_buffer.len, num_shards) catch unreachable; + + const shard_readers = try allocator.alloc(Io.ShardReader, num_shards); + for (shard_readers, 0.., input._shards.slice()) |*r, i, shard_buffer| { + r.* = .init( + input._api, + shard_buffer, + dma_buffer[i * dma_bytes_per_shard ..][0..dma_bytes_per_shard], + ); + } + + return .{ + .active = 0, + .shards = shard_readers, + .interface = .{ .vtable = &vtable, .buffer = &.{}, .seek = 0, .end = 0 }, + }; + } + + fn stream(r: *std.io.Reader, w: *std.io.Writer, limit: std.io.Limit) std.io.Reader.StreamError!usize { + // TODO(hugomano): this seem suboptimal, it only stream the first shard + // until end of stream, then switch to the second shard, ... + const self = @as(*Reader, @alignCast(@fieldParentPtr("interface", r))); + + while (self.active < self.shards.len) { + const active = self.active; + var shard_reader: *Io.ShardReader = &self.shards[active]; + const bytes_read = shard_reader.interface.stream(w, limit) catch |err| switch (err) { + error.EndOfStream => { + self.active += 1; + continue; + }, + else => |e| return e, + }; + return bytes_read; + } + return error.EndOfStream; + } + + const vtable: std.io.Reader.VTable = .{ + .stream = stream, + }; + }; }; const _MINOR_TO_MAJOR = blk: { diff --git a/zml/context.zig b/zml/context.zig index 27d635a0..e3627916 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -16,6 +16,7 @@ const zml = struct { pub const platform = @import("platform.zig"); pub const Shape = @import("shape.zig").Shape; pub const Target = @import("platform.zig").Target; + pub const tools = @import("tools/tools.zig"); }; const PjrtApiMap = std.EnumArray(zml.Target, ?*const pjrt.Api); diff --git a/zml/ops.zig b/zml/ops.zig index edbcd4a2..bd09ac4d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1213,9 +1213,13 @@ fn TensorOrTensorArray(comptime T: type) type { pub const CustomCallOptions = struct { has_side_effect: bool, output_operand_aliases: ?[]const i64 = null, + kernel_name: []const u8 = &.{}, + backend_config: []const u8 = &.{}, }; pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) TensorOrTensorArray(@TypeOf(outputs)) { + // TODO(corendos): metadata aren't used + _ = metadata; // Transform generic inputs to flat slice. const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { .@"struct" => |struct_info| b: { @@ -1262,7 +1266,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), }; - const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, metadata, opts); + const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, opts); // Transform flat slice to generic outputs. return switch (@typeInfo(@TypeOf(outputs))) { @@ -1288,7 +1292,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; } -fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { +fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, opts: CustomCallOptions) []Tensor { const ctx = CompilationContext.current(); const values = ctx.allocator().alloc(mlir.Value, inputs.len) catch unreachable; @@ -1299,21 +1303,6 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs res_types[i] = mlirx.tensorType(ctx.mlirCtx(), output); } - const metadata_type_info = @typeInfo(@TypeOf(metadata)); - var metadata_attributes_tuple: [metadata_type_info.@"struct".fields.len]mlir.AttrTuple = undefined; - inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { - const attribute: mlir.Attribute = switch (@typeInfo(field.type)) { - .int, .comptime_int => .int(ctx.mlirCtx(), .u64, @bitCast(@field(metadata, field.name))), - else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), - }; - metadata_attributes_tuple[i] = .{ field.name, attribute }; - } - - const backend_config = mlir.Attribute.dict(ctx.mlirCtx(), &(metadata_attributes_tuple ++ [_]mlir.AttrTuple{ - .{ "pjrt_api", .int(ctx.mlirCtx(), .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_api))) }, - .{ "pjrt_client", .int(ctx.mlirCtx(), .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_client))) }, - })); - const operands_layouts = ctx.allocator().alloc([]const usize, inputs.len) catch unreachable; for (inputs, 0..) |input, i| { operands_layouts[i] = minorToMajor(input.rank()); @@ -1329,7 +1318,8 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs values, .{ .call_target_name = target_name, - .backend_config = backend_config, + .kernel_name = opts.kernel_name, + .backend_config = .string(ctx.mlirCtx(), opts.backend_config), .has_side_effect = opts.has_side_effect, .api_version = .typed_ffi, .operand_layouts = operands_layouts, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index f09ffa1e..450f8976 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -137,6 +137,14 @@ pub const Client = opaque { pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); } + + pub fn dmaMap(self: *const Client, api: *const Api, ptr: []u8) ApiError!void { + return try self.inner().dmaMap(api, ptr); + } + + pub fn dmaUnmap(self: *const Client, api: *const Api, ptr: []u8) void { + return self.inner().dmaUnmap(api, ptr); + } }; pub const Buffer = opaque { @@ -194,6 +202,10 @@ pub const Buffer = opaque { return @ptrCast(try self.inner().copyToMemory(api, memory_)); } + pub fn copyRawToHost(self: *const Buffer, api: *const Api, dest: []u8, offset: i64) ApiError!?*Event { + return @ptrCast(try self.inner().copyRawToHost(api, dest, offset)); + } + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { return @ptrCast(self.inner().getReadyEvent(api)); } diff --git a/zml/shape.zig b/zml/shape.zig index 564f33b5..91225a9a 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -442,7 +442,7 @@ pub const Shape = struct { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); new_shape.inferMissingAxis(self.count()); - stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); + stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {f} to {f}", .{ self, new_shape }); return new_shape; } diff --git a/zml/tensor.zig b/zml/tensor.zig index d5acc541..c4347fcd 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2252,7 +2252,7 @@ pub const Tensor = struct { /// /// Some example input for the base case where we work on one axis: /// - gather(f:[a], .{ .a = idx:[n]})[n] == f[idx[n]] - /// - gather(f:[a, b], .a, idx:[n])[n, b] == f[idx[n], b] + /// - gather(f:[a, b], .{ .a = idx:[n]})[n, b] == f[idx[n], b] /// - gather(f:[a,b,c], .{.b = idx:[n,m]})[a, n, m, c] == f[a, idx[n, m], c] /// /// If an axis in common between `self` and `indices`, diff --git a/zml/zml.zig b/zml/zml.zig index 55559f5c..f5805437 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -29,6 +29,7 @@ pub const ModuleSignature = exe.ModuleSignature; pub const floats = @import("floats.zig"); pub const helpers = @import("helpers.zig"); pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; +pub const Io = @import("Io.zig"); pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig");