diff --git a/MODULE.bazel b/MODULE.bazel index 73468e4f..4c04352a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -21,7 +21,7 @@ bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") bazel_dep(name = "rules_rust", version = "0.63.0") -bazel_dep(name = "rules_zig", version = "0.12.2") +bazel_dep(name = "rules_zig", version = "0.12.3") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7a3758f1..6c24ff42 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -113,8 +113,8 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.12/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.14/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.8/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", @@ -176,7 +176,7 @@ "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_zig/0.12.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_zig/0.12.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_zig/0.12.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -333,9 +333,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", - "https://bcr.bazel.build/modules/rules_cc/0.2.11/MODULE.bazel": "e94f24f065bf2191dba2dace951814378b66a94bb3bcc48077492fe0508059b5", "https://bcr.bazel.build/modules/rules_cc/0.2.12/MODULE.bazel": "4216c383ce3223c7dfbd3afcb09c7056265156150b2a3bd3fd9d02949a7ee1bc", - "https://bcr.bazel.build/modules/rules_cc/0.2.12/source.json": "593d822a2ce7b70d5fad3195f7eb64d519dde94d7c20b0b79e6c23891a79b2bd", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/source.json": "55d0a4587c5592fad350f6e698530f4faf0e7dd15e69d43f8d87e220c78bea54", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", @@ -409,8 +409,8 @@ "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", "https://bcr.bazel.build/modules/rules_zig/0.12.1/MODULE.bazel": "5953094b681c212eebe3dd4275809c52b9177aeaa058d8f1bff433822c0dcd01", - "https://bcr.bazel.build/modules/rules_zig/0.12.2/MODULE.bazel": "09d2ca5486c5c3bbcaa61057958d35f9b4e1e981e08a5a28ad516bab87e8c4ef", - "https://bcr.bazel.build/modules/rules_zig/0.12.2/source.json": "4cef85ce1b625967ce9c063b8716920e8e7067c38321981a7bfd581ad588f4ba", + "https://bcr.bazel.build/modules/rules_zig/0.12.3/MODULE.bazel": "d2917ed1d6d0510553101be587bd2d2b236a63ac20612c8f26a7fcfb2fb5c47f", + "https://bcr.bazel.build/modules/rules_zig/0.12.3/source.json": "9b599ada6d04fbe0144b8e4092bdc5540d49b8c62a3a13988d6ab1f03ba69ae9", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -1169,7 +1169,7 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "ZaON2eA6LKrwcPU8xDGi0W3+6wD1r6kjb0pSYFC4Se8=", + "bzlTransitiveDigest": "zqRaSoEgRpMcDeWk1K16DqBxqQUILfmS+/URBwsw7CQ=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", @@ -2822,11 +2822,11 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "87SI2jgNGLvFKH9GAVUH7PVF1DJqfUvlXiSe9uiKbnk=", + "bzlTransitiveDigest": "EwflR/Okh5RJpz+iRzxWRcrX2K05f4LfBD8/pK+iE68=", + "usagesDigest": "Cw/jB/LASTlSFqF+jd3/TEs+kI2riEthW9Qh0bSKIoI=", "recordedFileInputs": { "@@//bazel/zig_index.json": "e6c6b29fa227fa9b1806dc03af2b6dab00f52f34d39878557074e1626027259f", - "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" + "@@rules_zig+//zig/private/versions.json": "12c90f64135026b686ef4299859baf0d9b285ef67eddcb21f2190333b915df4d" }, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/build.zig b/build.zig index e69de29b..05c29d08 100644 --- a/build.zig +++ b/build.zig @@ -0,0 +1,20 @@ +const std = @import("std"); + +/// Partial copy of: +/// https://github.com/zml/zml/pull/262 +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const test_step = b.step("test", "Run all tests across ZML and deps"); + + // stdx + const stdx = b.addModule("stdx", .{ + .root_source_file = b.path("stdx/stdx.zig"), + .target = target, + .optimize = optimize, + }); + + const run_stdx_tests = b.addRunArtifact(b.addTest(.{ .root_module = stdx })); + test_step.dependOn(&run_stdx_tests.step); +} diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 00000000..4bad5174 --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,61 @@ +.{ + .name = .zml, + .version = "0.0.0", + + .fingerprint = 0x32700c0d288fdb37, // Changing this has security and trust implications. + + .minimum_zig_version = "0.14.0", + + // This field is optional. + // Each dependency must either provide a `url` and `hash`, or a `path`. + // `zig build --fetch` can be used to fetch all dependencies of a package, recursively. + // Once all dependencies are fetched, `zig build` no longer requires + // internet connectivity. + .dependencies = .{ + .pjrt_cpu_darwin_arm64 = .{ + .url = "https://github.com/zml/pjrt-artifacts/releases/download/v10.0.0/pjrt-cpu_darwin-arm64.tar.gz", + .hash = "N-V-__8AADBK6gU12pCj6Yo3Q5zYgUefD3uH5kShV2pHGyOS", + .lazy = true, + }, + .pjrt_cpu_linux_amd64 = .{ + .url = "https://github.com/zml/pjrt-artifacts/releases/download/v10.0.0/pjrt-cpu_linux-amd64.tar.gz", + .hash = "N-V-__8AAAjaOQyQKeieiaG2csxnfXzIkaPNN4aX7p247PLE", + .lazy = true, + }, + }, + + // Specifies the set of files and directories that are included in this package. + // Only files and directories listed here are included in the `hash` that + // is computed for this package. Only files listed here will remain on disk + // when using the zig package manager. As a rule of thumb, one should list + // files required for compilation plus any license(s). + // Paths are relative to the build root. Use the empty string (`""`) to refer to + // the build root itself. + // A directory listed here means that all files within, recursively, are included. + .paths = .{ + "async", + "bazel", + "ffi", + "mlir", + "pjrt", + "platforms", + "runtimes", + "stdx", + "third_party", + "zml", + "build.zig", + "build.zig.zon", + // bazel files + "BUILD.bazel", + "MODULE.bazel", + "MODULE.bazel.lock", + // symlinks + ".bazelrc", + ".bazelrc-zml", + "examples/.bazelrc", + ".bazelignore", + "examples/.bazelignore", + ".bazelversion", + "examples/.bazelversion", + }, +} diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6d577695..457a751a 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -76,6 +76,10 @@ pub fn generateText( const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); defer allocator.free(prompt_tok); + const tracer: zml.tools.Tracer = .init("ai.zml.sampling"); + defer tracer.deinit(); + var inter_token_frame: zml.tools.Tracer.Frame = undefined; + var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); @@ -97,6 +101,8 @@ pub fn generateText( defer prefill_token_pos.deinit(); const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); + _ = try prefill_tokens.await(); + inter_token_frame = tracer.frameStart("inter_token"); _ = try prefilled_tokens.toHost(std.mem.sliceAsBytes(prefill_buffer)); generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; break :prefill kv_cache; @@ -139,8 +145,11 @@ pub fn generateText( defer token_pos.deinit(); // call to generate the next token + tracer.frameEnd(inter_token_frame, "inter_token"); current_token, kv_cache, rng = mod_generate.call(.{ current_token, token_pos, kv_cache, rng }); + _ = try current_token.await(); + inter_token_frame = tracer.frameStart("inter_token"); // extract the generated token from the buffer _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); } diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 7c7c1394..9a93a8ec 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -733,6 +733,7 @@ pub const CustomCallOpts = struct { }; call_target_name: [:0]const u8, + kernel_name: []const u8 = &.{}, has_side_effect: bool, backend_config: ?mlir.Attribute, operand_layouts: ?[]const []const usize = null, @@ -768,6 +769,7 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, .{ "backend_config", backend_config }, }); + if (opts.kernel_name.len > 0) attrs.appendAssumeCapacity(.{ "kernel_name", .string(ctx, opts.kernel_name) }); { var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index cad1ce69..8354ad62 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -440,18 +440,18 @@ pub const Client = opaque { } pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { - try api.call(.PJRT_Client_DmaMap, .{ + _ = try api.call(.PJRT_Client_DmaMap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), .size = @intCast(data.len), }); } - pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { - try api.call(.PJRT_Client_DmaUnmap, .{ + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) void { + _ = api.call(.PJRT_Client_DmaUnmap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), - }); + }) catch {}; } pub const CreateBuffersForAsyncHostToDeviceArgs = struct { diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index efe4abb0..089b3b64 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -37,6 +37,7 @@ zig_library( "floats.zig", "helpers.zig", "hostbuffer.zig", + "Io.zig", "meta.zig", "mlirx.zig", "module.zig", diff --git a/zml/Io.zig b/zml/Io.zig new file mode 100644 index 00000000..d65f5e85 --- /dev/null +++ b/zml/Io.zig @@ -0,0 +1,191 @@ +const builtin = @import("builtin"); +const std = @import("std"); + +const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); +const tracer = @import("context.zig").Context.tracer; +const log = std.log.scoped(.@"zml.io"); + +pub const ShardReader = struct { + pub const NUM_SLOTS = 2; + + pjrt_api: *const pjrt.Api, + pjrt_buffer: *const pjrt.Buffer, + total_size: u64, + chunk_size: usize, + + slots: [NUM_SLOTS]?*pjrt.Event, + dma_buffer: []u8, + bytes_requested: u64, + bytes_activated: u64, + next_request_slot: u1, + next_consume_slot: u1, + + is_primed: bool, + interface: std.io.Reader, + + pub fn init(pjrt_api: *const pjrt.Api, pjrt_buffer: *const pjrt.Buffer, dma_buffer: []u8) ShardReader { + const chunk_size = dma_buffer.len / NUM_SLOTS; + std.debug.assert(chunk_size > 0); + + return .{ + .pjrt_api = pjrt_api, + .pjrt_buffer = pjrt_buffer, + .total_size = pjrt_buffer.getOnDeviceSizeInBytes(pjrt_api) catch unreachable, + .chunk_size = chunk_size, + .slots = .{ null, null }, + .dma_buffer = dma_buffer, + .bytes_requested = 0, + .bytes_activated = 0, + .next_request_slot = 0, + .next_consume_slot = 0, + .is_primed = false, + .interface = .{ .vtable = &vtable, .buffer = &.{}, .seek = 0, .end = 0 }, + }; + } + + fn requestNextChunk(self: *ShardReader) std.io.Reader.StreamError!void { + const trace = tracer.frameStart("ShardReader.requestNextChunk"); + defer tracer.frameEnd(trace, "ShardReader.requestNextChunk"); + + const slot_to_fill = self.next_request_slot; + std.debug.assert(self.slots[slot_to_fill] == null); + + if (self.bytes_requested >= self.total_size) return; + + const offset_in_dma = slot_to_fill * self.chunk_size; + const chunk_dma_buf = self.dma_buffer[offset_in_dma .. offset_in_dma + self.chunk_size]; + const remaining_on_device = self.total_size - self.bytes_requested; + const transfer_size = @min(remaining_on_device, chunk_dma_buf.len); + const dest_slice = chunk_dma_buf[0..transfer_size]; + const offset: i64 = @intCast(self.bytes_requested); + + log.debug("ShardReader.requestNextChunk: slot={d}, device_offset={d}B, transfer_size={d}B, remaining={d}B", .{ + slot_to_fill, + offset, + transfer_size, + self.total_size - (self.bytes_requested + transfer_size), + }); + + const event = self.pjrt_buffer.copyRawToHost(self.pjrt_api, dest_slice, offset) catch |err| { + log.err("PJRT copyRawToHost failed: {}", .{err}); + return error.ReadFailed; + }; + + if (event) |ev| self.slots[slot_to_fill] = ev; + self.bytes_requested += transfer_size; + self.next_request_slot = 1 - self.next_request_slot; + } + + fn stream(r: *std.io.Reader, w: *std.io.Writer, limit: std.io.Limit) std.io.Reader.StreamError!usize { + const trace = tracer.frameStart("ShardReader.stream(vtable)"); + defer tracer.frameEnd(trace, "ShardReader.stream(vtable)"); + + // From Io.Writer.vtable documentation: + // > In addition to, or instead of writing to `w`, + // > the implementation may choose to store data in `buffer`. + // This is what we do since we need to write in DMA memory. + _ = w; + _ = limit; + const self = @as(*ShardReader, @alignCast(@fieldParentPtr("interface", r))); + + std.debug.assert(r.seek == r.end); + + if (!self.is_primed) { + self.is_primed = true; + if (self.total_size == 0) return error.EndOfStream; + log.debug("ShardReader: Priming pipeline for {d} bytes...", .{self.total_size}); + for (0..NUM_SLOTS) |_| try self.requestNextChunk(); + } else { + try self.requestNextChunk(); + } + + const slot_to_consume = self.next_consume_slot; + + if (self.slots[slot_to_consume]) |event| { + const trace_await = tracer.frameStart("ShardReader.awaitEvent"); + defer tracer.frameEnd(trace_await, "ShardReader.awaitEvent"); + + event.awaitBlocking(self.pjrt_api) catch |err| { + log.err("Error awaiting event in stream: {}", .{err}); + return error.ReadFailed; + }; + self.slots[slot_to_consume] = null; + } else { + if (self.bytes_activated >= self.total_size) { + return error.EndOfStream; + } else { + log.err("ShardReader stalled: waiting for slot {d} which has no event.", .{slot_to_consume}); + return error.ReadFailed; + } + } + + const offset_in_dma = slot_to_consume * self.chunk_size; + const remaining_total = self.total_size - self.bytes_activated; + const actual_chunk_size = @min(self.chunk_size, remaining_total); + + self.bytes_activated += actual_chunk_size; + + r.buffer = self.dma_buffer[offset_in_dma .. offset_in_dma + actual_chunk_size]; + r.seek = 0; + r.end = r.buffer.len; + + self.next_consume_slot = 1 - self.next_consume_slot; + + return 0; + } + + const vtable: std.io.Reader.VTable = .{ + .stream = stream, + }; +}; + +test "ShardReader: streamRemaining" { + const zml = @import("zml.zig"); + + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + + const trace_test = tracer.frameStart("ShardReader.test.streamRemaining"); + defer tracer.frameEnd(trace_test, "ShardReader.test.streamRemaining"); + + const Local = struct { + fn forward() zml.Tensor { + return .arange(.{ .end = 256 * MB }, .u32); + } + }; + + const x_d = try zml.testing.compileAndCall(platform, Local.forward, .{}); + defer x_d.deinit(); + + const dma_buffer = try allocator.alignedAlloc(u8, .fromByteUnits(4 * KB), 1 * MB); + defer allocator.free(dma_buffer); + + if (platform.target != .cpu) platform.pjrt_client.dmaMap(platform.pjrt_api, dma_buffer) catch {}; + defer if (platform.target != .cpu) platform.pjrt_client.dmaUnmap(platform.pjrt_api, dma_buffer); + + var device_reader: ShardReader = .init(platform.pjrt_api, x_d._shards.get(0), dma_buffer); + + const x_h = try allocator.alignedAlloc(u32, .fromByteUnits(16 * KB), x_d.shape().count()); + defer allocator.free(x_h); + var x_h_writer: std.Io.Writer = .fixed(@ptrCast(x_h)); + + const trace_stream = tracer.frameStart("ShardReader.streamRemaining"); + const bytes_read = try device_reader.interface.streamRemaining(&x_h_writer); + tracer.frameEnd(trace_stream, "ShardReader.streamRemaining"); + + std.log.warn("Device: {f}, host: {d}, read: {d}", .{ + x_d, + 256 * MB * @sizeOf(u32), + bytes_read, + }); + try std.testing.expectEqual(x_d.shape().byteSize(), bytes_read); + try std.testing.expectEqual(256 * MB * @sizeOf(u32), bytes_read); + for (x_h, 0..) |actual, expected| { + errdefer log.err("Mismatch at offset {d}, expected {x}, got {x}", .{ expected, expected, actual }); + try std.testing.expectEqual(expected, actual); + } +} + +const KB = 1024; +const MB = 1024 * KB; diff --git a/zml/buffer.zig b/zml/buffer.zig index e76039d2..dc5c15cc 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const DataType = @import("dtype.zig").DataType; const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const Io = @import("Io.zig"); const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -484,6 +485,62 @@ pub const Buffer = struct { } return deleted; } + + pub fn reader(self: Buffer, allocator: std.mem.Allocator, dma_buffer: []u8) !Reader { + return try .init(allocator, self, dma_buffer); + } + + const Reader = struct { + // TODO: optimize a bit, there is a lot of redundant information across shards. + shards: []Io.ShardReader, + active: u8, + interface: std.Io.Reader, + + pub fn init(allocator: std.mem.Allocator, input: Buffer, dma_buffer: []u8) !Reader { + const num_shards = input._shards.len; + std.debug.assert(num_shards > 0); + const dma_bytes_per_shard = std.math.divCeil(usize, dma_buffer.len, num_shards) catch unreachable; + + const shard_readers = try allocator.alloc(Io.ShardReader, num_shards); + for (shard_readers, 0.., input._shards.slice()) |*r, i, shard_buffer| { + r.* = .init( + input._api, + shard_buffer, + dma_buffer[i * dma_bytes_per_shard ..][0..dma_bytes_per_shard], + ); + } + + return .{ + .active = 0, + .shards = shard_readers, + .interface = .{ .vtable = &vtable, .buffer = &.{}, .seek = 0, .end = 0 }, + }; + } + + fn stream(r: *std.io.Reader, w: *std.io.Writer, limit: std.io.Limit) std.io.Reader.StreamError!usize { + // TODO(hugomano): this seem suboptimal, it only stream the first shard + // until end of stream, then switch to the second shard, ... + const self = @as(*Reader, @alignCast(@fieldParentPtr("interface", r))); + + while (self.active < self.shards.len) { + const active = self.active; + var shard_reader: *Io.ShardReader = &self.shards[active]; + const bytes_read = shard_reader.interface.stream(w, limit) catch |err| switch (err) { + error.EndOfStream => { + self.active += 1; + continue; + }, + else => |e| return e, + }; + return bytes_read; + } + return error.EndOfStream; + } + + const vtable: std.io.Reader.VTable = .{ + .stream = stream, + }; + }; }; const _MINOR_TO_MAJOR = blk: { diff --git a/zml/context.zig b/zml/context.zig index 27d635a0..e3627916 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -16,6 +16,7 @@ const zml = struct { pub const platform = @import("platform.zig"); pub const Shape = @import("shape.zig").Shape; pub const Target = @import("platform.zig").Target; + pub const tools = @import("tools/tools.zig"); }; const PjrtApiMap = std.EnumArray(zml.Target, ?*const pjrt.Api); diff --git a/zml/ops.zig b/zml/ops.zig index edbcd4a2..bd09ac4d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1213,9 +1213,13 @@ fn TensorOrTensorArray(comptime T: type) type { pub const CustomCallOptions = struct { has_side_effect: bool, output_operand_aliases: ?[]const i64 = null, + kernel_name: []const u8 = &.{}, + backend_config: []const u8 = &.{}, }; pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) TensorOrTensorArray(@TypeOf(outputs)) { + // TODO(corendos): metadata aren't used + _ = metadata; // Transform generic inputs to flat slice. const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { .@"struct" => |struct_info| b: { @@ -1262,7 +1266,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), }; - const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, metadata, opts); + const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, opts); // Transform flat slice to generic outputs. return switch (@typeInfo(@TypeOf(outputs))) { @@ -1288,7 +1292,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; } -fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { +fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, opts: CustomCallOptions) []Tensor { const ctx = CompilationContext.current(); const values = ctx.allocator().alloc(mlir.Value, inputs.len) catch unreachable; @@ -1299,21 +1303,6 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs res_types[i] = mlirx.tensorType(ctx.mlirCtx(), output); } - const metadata_type_info = @typeInfo(@TypeOf(metadata)); - var metadata_attributes_tuple: [metadata_type_info.@"struct".fields.len]mlir.AttrTuple = undefined; - inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { - const attribute: mlir.Attribute = switch (@typeInfo(field.type)) { - .int, .comptime_int => .int(ctx.mlirCtx(), .u64, @bitCast(@field(metadata, field.name))), - else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), - }; - metadata_attributes_tuple[i] = .{ field.name, attribute }; - } - - const backend_config = mlir.Attribute.dict(ctx.mlirCtx(), &(metadata_attributes_tuple ++ [_]mlir.AttrTuple{ - .{ "pjrt_api", .int(ctx.mlirCtx(), .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_api))) }, - .{ "pjrt_client", .int(ctx.mlirCtx(), .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_client))) }, - })); - const operands_layouts = ctx.allocator().alloc([]const usize, inputs.len) catch unreachable; for (inputs, 0..) |input, i| { operands_layouts[i] = minorToMajor(input.rank()); @@ -1329,7 +1318,8 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs values, .{ .call_target_name = target_name, - .backend_config = backend_config, + .kernel_name = opts.kernel_name, + .backend_config = .string(ctx.mlirCtx(), opts.backend_config), .has_side_effect = opts.has_side_effect, .api_version = .typed_ffi, .operand_layouts = operands_layouts, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index f09ffa1e..450f8976 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -137,6 +137,14 @@ pub const Client = opaque { pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); } + + pub fn dmaMap(self: *const Client, api: *const Api, ptr: []u8) ApiError!void { + return try self.inner().dmaMap(api, ptr); + } + + pub fn dmaUnmap(self: *const Client, api: *const Api, ptr: []u8) void { + return self.inner().dmaUnmap(api, ptr); + } }; pub const Buffer = opaque { @@ -194,6 +202,10 @@ pub const Buffer = opaque { return @ptrCast(try self.inner().copyToMemory(api, memory_)); } + pub fn copyRawToHost(self: *const Buffer, api: *const Api, dest: []u8, offset: i64) ApiError!?*Event { + return @ptrCast(try self.inner().copyRawToHost(api, dest, offset)); + } + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { return @ptrCast(self.inner().getReadyEvent(api)); } diff --git a/zml/platform.zig b/zml/platform.zig index c5872a57..1a36c46a 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -5,6 +5,7 @@ pub const Target = runtimes.Platform; const stdx = @import("stdx"); const pjrt = @import("pjrtx.zig"); +const tools = @import("tools/tools.zig"); const log = std.log.scoped(.zml); @@ -37,6 +38,15 @@ pub const Platform = struct { pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; pub const CreateOptions = _CreateOptions; + // Set by the first created Platform object. + pub var tracer: tools.Tracer = undefined; + + var tracer_once = std.once(struct { + fn call() void { + tracer = .init("ai.zml"); + } + }.call); + pub fn init(target: Target, api: *const pjrt.Api, options: CreateOptions) !Platform { var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); @@ -44,6 +54,7 @@ pub const Platform = struct { if (true_num_devices > MAX_NUM_DEVICES) { log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, true_num_devices, MAX_NUM_DEVICES }); } + tracer_once.call(); return .{ .target = target, .pjrt_api = api, diff --git a/zml/shape.zig b/zml/shape.zig index 564f33b5..91225a9a 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -442,7 +442,7 @@ pub const Shape = struct { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); new_shape.inferMissingAxis(self.count()); - stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); + stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {f} to {f}", .{ self, new_shape }); return new_shape; } diff --git a/zml/tensor.zig b/zml/tensor.zig index d5acc541..c4347fcd 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2252,7 +2252,7 @@ pub const Tensor = struct { /// /// Some example input for the base case where we work on one axis: /// - gather(f:[a], .{ .a = idx:[n]})[n] == f[idx[n]] - /// - gather(f:[a, b], .a, idx:[n])[n, b] == f[idx[n], b] + /// - gather(f:[a, b], .{ .a = idx:[n]})[n, b] == f[idx[n], b] /// - gather(f:[a,b,c], .{.b = idx:[n,m]})[a, n, m, c] == f[a, idx[n, m], c] /// /// If an axis in common between `self` and `indices`, diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 3946cb6c..213451f1 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -10,6 +10,7 @@ pub const Tracer = switch (builtin.os.tag) { }; const CudaTracer = struct { + pub const Frame = u64; // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). @@ -39,12 +40,12 @@ const CudaTracer = struct { nvtxMarkA(message.ptr); } - pub fn frameStart(self: *const CudaTracer, message: [:0]const u8) u64 { + pub fn frameStart(self: *const CudaTracer, message: [:0]const u8) Frame { _ = self; return @intCast(nvtxRangeStartA(message.ptr)); } - pub fn frameEnd(self: *const CudaTracer, interval_id: u64, message: [:0]const u8) void { + pub fn frameEnd(self: *const CudaTracer, interval_id: Frame, message: [:0]const u8) void { _ = self; _ = message; nvtxRangeEnd(@intCast(interval_id)); @@ -53,6 +54,7 @@ const CudaTracer = struct { }; const MacOsTracer = struct { + pub const Frame = c.os_signpost_id_t; logger: c.os_log_t, pub fn init(name: [:0]const u8) MacOsTracer { @@ -67,13 +69,13 @@ const MacOsTracer = struct { c.zml_os_signpost_event(self.logger, interval_id, message); } - pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { + pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) Frame { const interval_id = c.os_signpost_id_generate(self.logger); c.zml_os_signpost_interval_begin(self.logger, interval_id, message); return interval_id; } - pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { + pub fn frameEnd(self: *const MacOsTracer, interval_id: Frame, message: [:0]const u8) void { c.zml_os_signpost_interval_end(self.logger, interval_id, message); } }; diff --git a/zml/zml.zig b/zml/zml.zig index 55559f5c..7651ff25 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -29,6 +29,7 @@ pub const ModuleSignature = exe.ModuleSignature; pub const floats = @import("floats.zig"); pub const helpers = @import("helpers.zig"); pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; +pub const Io = @import("Io.zig"); pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig"); @@ -38,6 +39,7 @@ pub const call = ops.call; pub const pjrt = @import("pjrtx.zig"); pub const platform = @import("platform.zig"); pub const Platform = @import("platform.zig").Platform; +pub const tracer = Platform.tracer; pub const Shape = @import("shape.zig").Shape; pub const ShapeOf = @import("tensor.zig").ShapeOf; pub const Target = @import("platform.zig").Target;