From f09319bd095b89004ce62db9f904937a9a4bb69e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 7 Jan 2025 22:55:02 +0100 Subject: [PATCH 1/2] zml: make buffers and execute asynchronous This change makes buffer and execution completely asynchronous. This enable easy pipelining of multiple operations. It doesn't map well to `async` because the PJRT API is already asynchronous, so might as well use it as is. --- examples/mnist/mnist.zig | 2 +- pjrt/pjrt.zig | 2 +- zml/buffer.zig | 56 +++++++++++++++++++++------------------- zml/hostbuffer.zig | 19 ++++++++++++++ zml/pjrtx.zig | 25 +++++------------- 5 files changed, 57 insertions(+), 47 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c49300a9..96221ff1 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -105,7 +105,7 @@ pub fn asyncMain() !void { defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); + var result: zml.Buffer = mnist.call(.{try input.awaitt()}); defer result.deinit(); log.info( diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 2b0e65b6..ef27c0a1 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -818,7 +818,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, diff --git a/zml/buffer.zig b/zml/buffer.zig index e0829d39..0e74b946 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -90,38 +90,39 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - pjrt.Client.BufferFromHostBufferArgs{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - - frames.appendAssumeCapacity(frame); - } - - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } res._shards.appendAssumeCapacity(pjrt_buffer); } + return res; } + pub fn awaitt(self: Buffer) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.awaitt(self._api); + } + } + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -274,7 +275,7 @@ pub const Buffer = struct { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.awaitt(self._api); } return res; } @@ -285,10 +286,11 @@ pub const Buffer = struct { pub fn toHost(self: Buffer, output: []u8) !HostBuffer { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); + var hb = HostBuffer.fromBytes(self.shape(), output); + hb._event = maybe_event; + hb._api = self._api; + hb._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return hb; } /// Copies the content of the Buffer to the host. @@ -297,9 +299,9 @@ pub const Buffer = struct { const output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); - if (maybe_event) |event| { - try event.await_(self._api); - } + output._event = maybe_event; + output._api = self._api; + output._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; return output; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 1885a001..ab6847d6 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -2,6 +2,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); + const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -23,6 +25,9 @@ pub const HostBuffer = struct { managed: std.mem.Alignment, unmanaged, } = .unmanaged, + _ready: bool = true, + _api: ?*const pjrt.Api = null, + _event: ?*pjrt.Event = null, /// Allocates a HostBuffer with the given shape. /// The memory is left undefined. @@ -269,6 +274,20 @@ pub const HostBuffer = struct { }; } + pub fn awaitt(self: *HostBuffer) !*HostBuffer { + if (self._ready) { + return self; + } + if (self._event) |ev| { + if (self._api) |api| { + try ev.awaitt(api); + } else { + stdx.debug.panic("Got an event but no API. This should not happen.", .{}); + } + } + return self; + } + pub fn format( self: HostBuffer, comptime fmt: []const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index bd094ead..ba0665a0 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -57,13 +57,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return @ptrCast(buffer); + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -153,10 +149,6 @@ pub const Buffer = opaque { return self.inner().isOnCpu(api); } - pub fn memory(self: *const Buffer, api: *const Api) *const Memory { - return self.inner().memory(api); - } - pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { return @ptrCast(try self.inner().toHostBuffer(api, dst)); } @@ -181,10 +173,6 @@ pub const Buffer = opaque { return @ptrCast(self.inner().copyToDevice(api, device)); } - pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(self.inner().copyToMemory(api, memory_)); - } - pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { return @ptrCast(self.inner().getReadyEvent(api)); } @@ -209,7 +197,8 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) ApiError!void { + pub const await_ = awaitt; + pub fn awaitt(self: *Event, api: *const Api) !void { defer self.deinit(api); if (self.isReady(api)) { @@ -265,13 +254,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From a63beea709a9f216704eeef78ec6daeede5f0e2a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 15 May 2025 09:57:39 +0000 Subject: [PATCH 2/2] WIP --- examples/MODULE.bazel | 1 + examples/MODULE.bazel.lock | 3 + examples/benchmark/main.zig | 39 ++++-- examples/mnist/mnist.zig | 35 +++-- pjrt/pjrt.zig | 265 ++++++++++++++++++++++++++---------- stdx/time.zig | 4 + zml/buffer.zig | 44 ++++-- zml/exe.zig | 11 +- zml/module.zig | 2 +- zml/pjrtx.zig | 33 ++++- zml/platform.zig | 4 +- zml/shape.zig | 10 ++ zml/tensor.zig | 2 +- 13 files changed, 324 insertions(+), 129 deletions(-) diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index eec14e80..4352e3e2 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -7,6 +7,7 @@ bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_rust", version = "0.60.0") bazel_dep(name = "rules_zig", version = "20250314.0-b9739c6") bazel_dep(name = "zml", version = "0.1.0") +bazel_dep(name = "mimalloc", version = "2.1.7-bcr.alpha.1") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") use_repo(non_module_deps, "com_github_hejsil_clap") diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 1008eb86..8b43dcea 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -56,6 +56,7 @@ "https://bazel-registry.zml.ai/modules/hermetic_cc_toolchain/3.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/libpfm/4.11.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/mimalloc/2.1.7-bcr.alpha.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/patchelf/0.18.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pcre2/10.43/MODULE.bazel": "not found", @@ -236,6 +237,8 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/mimalloc/2.1.7-bcr.alpha.1/MODULE.bazel": "b132ac74863a164e5b078940352ae2b39bf6dd52b7fe71caffc3d810db8b3de7", + "https://bcr.bazel.build/modules/mimalloc/2.1.7-bcr.alpha.1/source.json": "413edf3ca260ef2ca0994c560fe84e612ba82a40f024af939166e3571b5b27ca", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/patchelf/0.18.0/MODULE.bazel": "15a6beff7e828d585c5bd0f9f93589df117b5594e9d19e43096c77de58b9ae5f", diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index ac39ed1a..3c1e563f 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -6,7 +6,7 @@ const flags = stdx.flags; // set log level to debug to print the generated IR pub const std_options: std.Options = .{ - .log_level = .warn, + .log_level = .info, .logFn = asynk.logFn(std.log.defaultLog), }; @@ -37,7 +37,7 @@ pub fn asyncMain() !void { // Auto-select platform const platform = context.autoPlatform(.{}).withCompilationOptions(.{ - .sharding_enabled = true, + .sharding_enabled = false, }); context.printAvailablePlatforms(platform); @@ -46,7 +46,7 @@ pub fn asyncMain() !void { const a_shape = zml.Shape.init(.{ cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .m, .k }).withSharding(.{.k}); const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); - var timer = try std.time.Timer.start(); + var timer: stdx.time.Timer = try .start(); std.debug.print("\nCompiling model to MLIR....\n", .{}); std.debug.print("-" ** 160 ++ "\n", .{}); @@ -58,16 +58,19 @@ pub fn asyncMain() !void { // Wait for compilation to finish const executable = try compilation.awaitt(); defer executable.deinit(); - const compilation_elapsed = timer.lap() / std.time.ns_per_ms; + const compilation_elapsed = timer.lap(); std.debug.print("-" ** 160 ++ "\n\n", .{}); - std.debug.print("✅ Compiled Benchmark model in {d} milliseconds! \n", .{compilation_elapsed}); + std.debug.print("✅ Compiled Benchmark model in {}! \n", .{compilation_elapsed}); var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); var a_buffer = try createRandomBuffer(allocator, platform, a_shape, random); + // _ = try a_buffer.awaitt(); defer a_buffer.deinit(); + std.debug.print("2 \n", .{}); var b_buffer = try createRandomBuffer(allocator, platform, b_shape, random); + // _ = try b_buffer.awaitt(); defer b_buffer.deinit(); std.debug.print("\nRunning benchmark....\n", .{}); @@ -75,22 +78,32 @@ pub fn asyncMain() !void { // Ignore first run { var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + // var pouet = try result.toMemory(.host_pinned); + // _ = try pouet.awaitt(); defer result.deinit(); + // defer pouet.deinit(); } + try asynk.sleep(3000); + // call our executable module timer.reset(); - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); - defer result.deinit(); - const elapsed_ns = timer.lap(); - const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; - const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; + var result = a_buffer; + for (0..10) |_| { + result = executable.call(.{ result, b_buffer }); + _ = try result.awaitt(); + // var pouet = try result.toMemory(.host_pinned); + // pouet = pouet; + } + + // defer result.deinit(); + const elapsed = timer.lap(); std.debug.print("\n✅ Benchmark done!\n\n", .{}); const floating_op_count = 2 * cli_args.size * cli_args.size * cli_args.size; - const flops = @as(f64, @floatFromInt(floating_op_count)) / elapsed_s; - std.debug.print("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {d:.3}ms - {d:.3} GFLOP/s\n\n", .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), elapsed_ms, flops / 1_000_000_000 }); + const flops: f64 = @floatFromInt(elapsed.hz() * floating_op_count); + std.debug.print("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {} - {d:.3} GFLOP/s\n\n", .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), elapsed, flops / 1e9 }); } fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { @@ -123,5 +136,7 @@ fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shap var host_buffer = zml.HostBuffer.fromBytes(shape, data); errdefer host_buffer.deinit(allocator); + // var b = zml.Buffer.asViewOfHostBuffer(platform, host_buffer); + // return try b.toMemory(.device); return zml.Buffer.from(platform, host_buffer); } diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 96221ff1..7652edbc 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -96,25 +96,30 @@ pub fn asyncMain() !void { defer dataset.close() catch unreachable; var rng = std.Random.Xoshiro256.init(@intCast(std.time.timestamp())); + const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); + const sample = try std.heap.c_allocator.alloc(u8, 28 * 28); + _ = try dataset.pread(sample, 16 + (idx * 28 * 28)); + var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), sample)); + defer input.deinit(); + + for (0..10) |_| // inference - can be looped { - const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); - var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); - defer input.deinit(); - printDigit(sample); - var result: zml.Buffer = mnist.call(.{try input.awaitt()}); - defer result.deinit(); + // var sample: [28 * 28]u8 align(32) = undefined; - log.info( - \\✅ RECOGNIZED DIGIT: - \\ +-------------+ - \\{s} - \\ +-------------+ - \\ - , .{digits[try result.getValue(u8)]}); + // printDigit(sample[0 .. 28 * 28].*); + var result: zml.Buffer = mnist.call(.{input}); + result = result; // autofix + // defer result.deinit(); + + // log.info( + // \\✅ RECOGNIZED DIGIT: + // \\ +-------------+ + // \\{s} + // \\ +-------------+ + // \\ + // , .{digits[try result.getValue(u8)]}); } } diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index ef27c0a1..2501e9df 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -108,7 +108,7 @@ pub const Api = struct { return arg_array_type_info.pointer.child; } - inline fn call(self: *const Api, comptime method: Funcs, arg: CallFnArgType(method)) ApiError!@TypeOf(arg) { + fn call(self: *const Api, comptime method: Funcs, arg: CallFnArgType(method)) ApiError!@TypeOf(arg) { var ret = pjrtStruct(arg); const fn_ptr = @field(&self.inner, @tagName(method)).?; const result = fn_ptr(&ret); @@ -227,6 +227,27 @@ pub const ErrorCode = enum(c.PJRT_Error_Code) { .unauthenticated => ApiError.Unauthenticated, }; } + + pub fn fromApiError(err: ApiError) ErrorCode { + return switch (err) { + ApiError.Cancelled => .cancelled, + ApiError.Unknown => .unknown, + ApiError.InvalidArgument => .invalid_argument, + ApiError.DeadlineExceeded => .deadline_exceeded, + ApiError.NotFound => .not_found, + ApiError.AlreadyExists => .already_exists, + ApiError.PermissionDenied => .permission_denied, + ApiError.ResourceExhausted => .resource_exhausted, + ApiError.FailedPrecondition => .failed_precondition, + ApiError.Aborted => .aborted, + ApiError.OutOfRange => .out_of_range, + ApiError.Unimplemented => .unimplemented, + ApiError.Internal => .internal, + ApiError.Unavailable => .unavailable, + ApiError.DataLoss => .data_loss, + ApiError.Unauthenticated => .unauthenticated, + }; + } }; pub const Error = opaque { @@ -336,6 +357,7 @@ pub const Client = opaque { pub fn compile(self: *const Client, api: *const Api, args: CompileArgs) ApiError!*LoadedExecutable { const bytecode_format_ = @tagName(args.bytecode_format); const ret = try api.call(.PJRT_Client_Compile, .{ + .client = self.inner(), .program = &pjrtStruct(c.PJRT_Program{ .code = @ptrCast(@constCast(args.bytecode.ptr)), .code_size = args.bytecode.len, @@ -344,7 +366,6 @@ pub const Client = opaque { }), .compile_options = @ptrCast(@constCast(args.compile_options_pb.ptr)), .compile_options_size = args.compile_options_pb.len, - .client = self.inner(), }); return @ptrCast(ret.executable.?); } @@ -353,31 +374,31 @@ pub const Client = opaque { data: []const u8, buffer_type: BufferType, dims: []const i64, - byte_strides: ?[]const i64, + byte_strides: []const i64 = &.{}, device: ?*const Device = null, host_buffer_semantics: HostBufferSemantics, memory: ?*const Memory = null, + device_layout: MemoryLayout, }; pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { + const layout = args.device_layout.to_c(); const ret = try api.call(.PJRT_Client_BufferFromHostBuffer, .{ .client = self.inner(), .data = @ptrCast(@constCast(args.data.ptr)), .type = @intFromEnum(args.buffer_type), .dims = @ptrCast(@constCast(args.dims.ptr)), .num_dims = args.dims.len, - .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs.ptr)) else null, - .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, + .byte_strides = if (args.byte_strides.len > 0) @ptrCast(@constCast(args.byte_strides.ptr)) else null, + .num_byte_strides = args.byte_strides.len, .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), .device = @ptrCast(@constCast(args.device)), .memory = @ptrCast(@constCast(args.memory)), - .device_layout = null, // TODO - .done_with_host_buffer = null, - .buffer = null, + .device_layout = @ptrCast(@constCast(&layout)), }); return .{ - @ptrCast(ret.buffer.?), + @ptrCast(ret.buffer), @ptrCast(ret.done_with_host_buffer), }; } @@ -409,16 +430,16 @@ pub const Client = opaque { dims: []const i64, element_type: BufferType, layout: MemoryLayout, - device: *const Device, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.C) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.C) void {} }.call, on_delete_callback_arg: ?*anyopaque = null, stream: ?isize = null, + memory: ?*const Memory = null, }; pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { - const layout = args.layout.toCStruct(); + const layout = args.layout.to_c(); const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), .device_buffer_ptr = @ptrCast(@constCast(args.data)), @@ -426,10 +447,10 @@ pub const Client = opaque { .num_dims = args.dims.len, .element_type = @intFromEnum(args.element_type), .layout = @ptrCast(@constCast(&layout)), - .device = @ptrCast(@constCast(args.device)), .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, .stream = if (args.stream) |stream| stream else 0, + .memory = @ptrCast(@constCast(args.memory)), }); return @ptrCast(ret.buffer.?); } @@ -444,25 +465,26 @@ pub const Client = opaque { return &.{}; } - pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!*Buffer { - const ret = try api.call(.PJRT_Client_DMA_Map, .{ + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + const args: Api.CallFnArgType(.PJRT_Client_DmaMap) = .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), .size = @intCast(data.len), - }); - return @ptrCast(ret.buffer.?); + }; + std.debug.print("DMA map {any} \n", .{args}); + _ = try api.call(.PJRT_Client_DmaMap, args); } - pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) void { - _ = api.call(.PJRT_Client_DMA_Unmap, .{ + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + _ = try api.call(.PJRT_Client_DmaUnmap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), - }) catch unreachable; + }); } pub const CreateBuffersForAsyncHostToDeviceArgs = struct { shape_specs: []const ShapeSpec, - device_layouts: ?[]*const MemoryLayout = null, + device_layouts: []*const MemoryLayout = &.{}, memory: *const Memory, }; @@ -471,11 +493,18 @@ pub const Client = opaque { .client = self.inner(), .shape_specs = @ptrCast(args.shape_specs.ptr), .num_shape_specs = args.shape_specs.len, - .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts.ptr)) else null, - .num_device_layouts = if (args.device_layouts) |layouts| @intCast(layouts.len) else 0, + .device_layouts = if (args.device_layouts.len > 0) @ptrCast(@constCast(args.device_layouts)) else null, + .num_device_layouts = @intCast(args.device_layouts.len), .memory = @ptrCast(@constCast(args.memory)), }); - return @ptrCast(ret.transfer_manager.?); + return @ptrCast(ret.transfer_manager); + } + + pub fn topologyDescription(self: *const Client, api: *const Api) *const TopologyDescription { + const ret = api.call(.PJRT_Client_TopologyDescription, .{ + .client = self.inner(), + }) catch unreachable; + return @ptrCast(ret.topology); } }; @@ -503,12 +532,21 @@ pub const Device = opaque { return @intCast(ret.local_hardware_id); } - pub fn addressableMemories(self: *const Device, api: *const Api) ApiError![]const *Memory { - const ret = try api.call(.PJRT_Device_AddressableMemories, .{ + pub fn addressableMemories(self: *const Device, api: *const Api) []const *Memory { + const ret = api.call(.PJRT_Device_AddressableMemories, .{ .device = self.inner(), - }); + }) catch unreachable; return @ptrCast(ret.memories[0..ret.num_memories]); } + + pub fn addressableMemory(self: *const Device, api: *const Api, kind: Memory.Kind) ?*const Memory { + for (self.addressableMemories(api)) |mem| { + if (std.mem.eql(u8, mem.kindStr(api), @tagName(kind))) { + return @ptrCast(mem); + } + } + return null; + } }; pub const DeviceDescription = opaque { @@ -546,7 +584,14 @@ pub const DeviceDescription = opaque { const ret = api.call(.PJRT_DeviceDescription_ToString, .{ .device_description = self.inner(), }) catch unreachable; - return ret.to_string[0..ret.to_string_size]; + return ret.to_string[0..ret.num_attributes]; + } + + pub fn attributes(self: *const DeviceDescription, api: *const Api) []const NamedValue { + const ret = api.call(.PJRT_DeviceDescription_Attributes, .{ + .device_description = self.inner(), + }) catch unreachable; + return @ptrCast(ret.attributes[0..ret.num_attributes]); } }; @@ -648,7 +693,7 @@ pub const LoadedExecutable = opaque { .num_devices = @intCast(args.arguments.len), .num_args = args.num_args, .output_lists = @ptrCast(args.results.ptr), - .device_complete_events = @ptrCast(args.events.ptr), + .device_complete_events = if (args.events.len > 0) @ptrCast(args.events.ptr) else null, .execute_device = null, }); } @@ -708,7 +753,7 @@ pub const MemoryLayout = union(MemoryLayoutType) { tiled: Tiled, strides: Strides, - fn toCStruct(self: MemoryLayout) c.PJRT_Buffer_MemoryLayout { + fn to_c(self: MemoryLayout) c.PJRT_Buffer_MemoryLayout { return pjrtStruct(switch (self) { .tiled => |v| c.PJRT_Buffer_MemoryLayout{ .type = c.PJRT_Buffer_MemoryLayout_Type_Tiled, @@ -818,7 +863,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, @@ -931,9 +976,12 @@ pub const Memory = opaque { } pub fn kind(self: *const Memory, api: *const Api) Kind { + return std.meta.stringToEnum(Kind, self.kindStr(api)) orelse unreachable; + } + + fn kindStr(self: *const Memory, api: *const Api) []const u8 { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable[0..ret.kind_size]; - return std.meta.stringToEnum(Kind, kind_) orelse unreachable; + return ret.kind[0..ret.kind_size]; } pub fn kindId(self: *const Memory, api: *const Api) u32 { @@ -992,7 +1040,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { .transfer_size = @intCast(data.len), .is_last_transfer = is_last_transfer, }); - return @ptrCast(ret.done_with_h2d_transfer.?); + return @ptrCast(ret.done_with_h2d_transfer); } pub fn retrieveBuffer(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!*Buffer { @@ -1000,20 +1048,20 @@ pub const AsyncHostToDeviceTransferManager = opaque { .transfer_manager = self.inner(), .buffer_index = @intCast(buffer_index), }); - return @ptrCast(ret.buffer_out.?); + return @ptrCast(ret.buffer_out); } - pub fn device(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!*Device { + pub fn device(self: *AsyncHostToDeviceTransferManager, api: *const Api) *Device { const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_Device, .{ .transfer_manager = self.inner(), - }); - return @ptrCast(ret.device_out.?); + }) catch unreachable; + return @ptrCast(ret.device_out); } - pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!usize { + pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) usize { const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_BufferCount, .{ .transfer_manager = self.inner(), - }); + }) catch unreachable; return ret.buffer_count; } @@ -1025,22 +1073,22 @@ pub const AsyncHostToDeviceTransferManager = opaque { return ret.buffer_size; } - pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: c.PJRT_Error_Code, error_message: []const u8) ApiError!void { + pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, err: ApiError, error_message: []const u8) ApiError!void { _ = try api.call(.PJRT_AsyncHostToDeviceTransferManager_SetBufferError, .{ .transfer_manager = self.inner(), .buffer_index = @intCast(buffer_index), - .error_code = error_code, - .error_message = error_message.ptr, + .error_code = ErrorCode.fromApiError(err), + .error_message = @ptrCast(error_message), .error_message_size = error_message.len, }); } - pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) ApiError!void { + pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) void { _ = try api.call(.PJRT_AsyncHostToDeviceTransferManager_AddMetadata, .{ .transfer_manager = self.inner(), - .transfer_metadata = transfer_metadata.ptr, + .transfer_metadata = @ptrCast(transfer_metadata), .num_metadata = transfer_metadata.len, - }); + }) catch unreachable; } }; @@ -1083,66 +1131,93 @@ pub const NamedValue = extern struct { } pub fn from(name_: []const u8, value: anytype) NamedValue { - return switch (@TypeOf(value)) { + const T = @TypeOf(value); + return switch (T) { []u8, []const u8 => fromString(name_, value), - i64 => fromInt64(name_, value), - []i64, []const i64 => fromInt64List(name_, value), - f32 => fromFloat(name_, value), bool => fromBool(name_, value), - else => fromString(name_, @tagName(value)), + f32 => fromFloat(name_, value), + []i64, []const i64 => fromInt64List(name_, value), + @TypeOf(.enum_literal) => fromString(name_, @tagName(value)), + else => switch (@typeInfo(T)) { + .int, .comptime_int => fromInt64(name_, @intCast(value)), + .@"enum", .@"union" => fromString(name_, @tagName(value)), + else => unreachable, + }, }; } - pub fn fromString(name_: []const u8, value: []const u8) NamedValue { + pub fn fromString(name_: []const u8, value_: []const u8) NamedValue { return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ .name = @ptrCast(@constCast(name_.ptr)), .name_size = name_.len, .type = c.PJRT_NamedValue_kString, - .unnamed_0 = .{ .string_value = @ptrCast(@constCast(value.ptr)) }, - .value_size = value.len, + .unnamed_0 = .{ .string_value = @ptrCast(@constCast(value_.ptr)) }, + .value_size = value_.len, }) }; } - pub fn fromInt64(name_: []const u8, value: i64) NamedValue { + pub fn fromInt64(name_: []const u8, value_: i64) NamedValue { return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ .name = @ptrCast(@constCast(name_.ptr)), .name_size = name_.len, .type = c.PJRT_NamedValue_kInt64, - .unnamed_0 = .{ .int64_value = value }, + .unnamed_0 = .{ .int64_value = value_ }, .value_size = 1, }) }; } - pub fn fromInt64List(name_: []const u8, value: []const i64) NamedValue { + pub fn fromInt64List(name_: []const u8, value_: []const i64) NamedValue { return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ .name = @ptrCast(@constCast(name_.ptr)), .name_size = name_.len, .type = c.PJRT_NamedValue_kInt64List, - .unnamed_0 = .{ .int64_array_value = @ptrCast(@constCast(value.ptr)) }, - .value_size = value.len, + .unnamed_0 = .{ .int64_array_value = @ptrCast(@constCast(value_.ptr)) }, + .value_size = value_.len, }) }; } - pub fn fromFloat(name_: []const u8, value: f32) NamedValue { + pub fn fromFloat(name_: []const u8, value_: f32) NamedValue { return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ .name = @ptrCast(@constCast(name_.ptr)), .name_size = name_.len, .type = c.PJRT_NamedValue_kFloat, - .unnamed_0 = .{ .float_value = value }, + .unnamed_0 = .{ .float_value = value_ }, .value_size = 1, }) }; } - pub fn fromBool(name_: []const u8, value: bool) NamedValue { + pub fn fromBool(name_: []const u8, value_: bool) NamedValue { return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ .name = @ptrCast(@constCast(name_.ptr)), .name_size = name_.len, .type = c.PJRT_NamedValue_kBool, - .unnamed_0 = .{ .bool_value = value }, + .unnamed_0 = .{ .bool_value = value_ }, .value_size = 1, }) }; } + pub fn string(self: NamedValue) []const u8 { + const u = self.inner.unnamed_0; + return u.string_value[0..self.inner.value_size]; + } + + pub fn int64list(self: NamedValue) []const i64 { + const u = self.inner.unnamed_0; + return u.int64_array_value[0..self.inner.value_size]; + } + + pub fn int64(self: NamedValue) i64 { + return self.inner.unnamed_0.int64_value; + } + + pub fn float(self: NamedValue) f32 { + return self.inner.unnamed_0.float_value; + } + + pub fn bool_(self: NamedValue) bool { + return self.inner.unnamed_0.bool_value; + } + pub fn format( self: NamedValue, comptime fmt: []const u8, @@ -1151,16 +1226,15 @@ pub const NamedValue = extern struct { ) !void { _ = fmt; _ = options; - try writer.print("{s}{{ .name = {s},", .{ @typeName(NamedValue), self.inner.name[0..self.inner.name_size] }); - const u = self.inner.unnamed_0; + try writer.print("{}{{ .name = {s}, .{s} = ", .{ NamedValue, self.name(), @tagName(self.kind()) }); switch (self.kind()) { - .string => try writer.print(" .string = {s} ", .{u.string_value[0..self.inner.value_size]}), - .int64 => try writer.print(" .int64 = {d} ", .{u.int64_value}), - .int64list => try writer.print(" .int64list = {d} ", .{u.int64_array_value[0..self.inner.value_size]}), - .float => try writer.print(" .float = {d} ", .{u.float_value}), - .bool => try writer.print(" .bool = {} ", .{u.bool_value}), + .string => try writer.print("\"{s}\"", .{self.string()}), + .int64 => try writer.print("{d}", .{self.int64()}), + .int64list => try writer.print("{d}", .{self.int64list()}), + .float => try writer.print("{d}", .{self.float()}), + .bool => try writer.print("{}", .{self.bool_()}), } - try writer.writeAll("}"); + try writer.writeAll(" }"); } }; @@ -1192,3 +1266,54 @@ pub const CustomCallRegistry = extern struct { } } }; + +pub const TopologyDescription = opaque { + const inner = InnerMixin(c.PJRT_TopologyDescription).inner; + + pub const InitArgs = struct { + topology_name: []const u8, + create_options: []const NamedValue, + }; + + pub fn init(api: *const Api, args: InitArgs) !*const TopologyDescription { + const ret = api.call(.PJRT_TopologyDescription_Create, .{ + .topology_name = @ptrCast(args.topology_name), + .topology_name_size = args.topology_name.len, + .create_options = @ptrCast(args.create_options), + .num_options = args.create_options.len, + }); + return @ptrCast(ret.topology); + } + + pub fn deinit(self: *TopologyDescription, api: *const Api) void { + _ = api.call(.PJRT_TopologyDescription_Destroy, .{ + .topology = self.inner(), + }) catch unreachable; + } + + pub fn platformVersion(self: *const TopologyDescription, api: *const Api) []const u8 { + const ret = api.call(.PJRT_TopologyDescription_PlatformVersion, .{ .topology = self.inner() }) catch unreachable; + return @ptrCast(ret.platform_version[0..ret.platform_version_size]); + } + + pub fn platformName(self: *const TopologyDescription, api: *const Api) []const u8 { + const ret = api.call(.PJRT_TopologyDescription_PlatformName, .{ .topology = self.inner() }) catch unreachable; + return @ptrCast(ret.platform_name[0..ret.platform_name_size]); + } + + pub fn getDeviceDescriptions(self: *const TopologyDescription, api: *const Api) []const DeviceDescription { + const ret = api.call(.PJRT_TopologyDescription_GetDeviceDescriptions, .{ .topology = self.inner() }) catch unreachable; + if (ret.num_descriptions == 0) { + return &.{}; + } + return @ptrCast(ret.descriptions[0..ret.num_descriptions]); + } + + pub fn attributes(self: *const TopologyDescription, api: *const Api) []const NamedValue { + const ret = api.call(.PJRT_TopologyDescription_Attributes, .{ .topology = self.inner() }) catch unreachable; + if (ret.num_attributes == 0) { + return &.{}; + } + return @ptrCast(@constCast(ret.attributes[0..ret.num_attributes])); + } +}; diff --git a/stdx/time.zig b/stdx/time.zig index e0759ab7..b77a489e 100644 --- a/stdx/time.zig +++ b/stdx/time.zig @@ -7,6 +7,10 @@ pub const Duration = struct { return .{ .ns = self.ns / rhs }; } + pub fn hz(self: Duration) u64 { + return (1 * std.time.ns_per_s) / self.ns; + } + pub fn format( self: Duration, comptime fmt: []const u8, diff --git a/zml/buffer.zig b/zml/buffer.zig index 0e74b946..ae89e73d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -31,7 +31,7 @@ pub const Buffer = struct { host_pinned, device, - pub fn toPjrtMemory(self: Memory) pjrt.Memory.Kind { + pub fn toPjrt(self: Memory) pjrt.Memory.Kind { return switch (self) { .host => .unpinned_host, .host_pinned => .pinned_host, @@ -39,8 +39,12 @@ pub const Buffer = struct { }; } - pub fn pjrtName(self: Memory) []const u8 { - return @tagName(self.toPjrtMemory()); + pub fn fromPjrt(m: pjrt.Memory.Kind) Memory { + return switch (m) { + .unpinned_host => .host, + .pinned_host => .host_pinned, + .device => .device, + }; } }; @@ -88,7 +92,6 @@ pub const Buffer = struct { } else 0; const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); - const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); const devices = platform.getDevices(); for (0..n_partitions) |i| { @@ -101,9 +104,15 @@ pub const Buffer = struct { .data = buf.data, .buffer_type = buffer_type, .dims = buf.shape().dims(), - .byte_strides = byte_strides, .device = devices[i], .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .device_layout = .{ + .tiled = .{ + .minor_to_major = host_buffer.shape().minor_to_major(), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, }); if (event) |ev| { ev.deinit(platform.pjrt_api); @@ -114,7 +123,7 @@ pub const Buffer = struct { return res; } - pub fn awaitt(self: Buffer) !Buffer { + pub fn awaitt(self: *Buffer) !*Buffer { for (self._shards.constSlice()) |buffer| { if (buffer.getReadyEvent(self._api)) |ev| { try ev.awaitt(self._api); @@ -243,12 +252,14 @@ pub const Buffer = struct { break :blk res; }; + const device = platform.getDevices()[0]; + const memory = device.addressableMemory(platform.pjrt_api, .pinned_host) orelse unreachable; + const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, .element_type = bufferTypeFromDtype(shape_.dtype()), .dims = shape_.dims(), // TODO: exposes sharding in the API. - .device = platform.getDevices()[0], .layout = .{ .tiled = .{ .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], @@ -257,7 +268,8 @@ pub const Buffer = struct { }, }, .stream = @bitCast(@as(usize, @intFromPtr(stream))), - }) catch @panic("failed to createViewOfDeviceBuffer"); + .memory = memory, + }) catch stdx.debug.panic("failed to createViewOfDeviceBuffer", .{}); var shards: Shards = .{}; shards.appendAssumeCapacity(pjrt_buffer); @@ -280,6 +292,16 @@ pub const Buffer = struct { return res; } + pub fn toMemory(self: *Buffer, memory: Memory) !Buffer { + var new_buffer = self.*; + const api = self._api; + for (self._shards.slice(), new_buffer._shards.slice()) |old_shard, *new_shard| { + const device = try old_shard.getDevice(api); + new_shard.* = try old_shard.copyToMemory(api, device.addressableMemory(api, memory.toPjrt()) orelse unreachable); + } + return new_buffer; + } + /// Copies the content of the Buffer back to host, in the given buffer, /// and return a new `HostBuffer` object with the same shape. /// The returned `HostBuffer` doesn't own the memory. @@ -296,7 +318,7 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); + var output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); output._event = maybe_event; @@ -354,9 +376,9 @@ pub const Buffer = struct { try writer.print("Buffer({_})", .{self._shape}); } - pub fn getMemory(self: Buffer) *const pjrt.Memory { + pub fn getMemory(self: Buffer) Memory { const shard = self._shards.get(0); - return shard.memory(self._api); + return .fromPjrt(shard.memory(self._api).kind(self._api)); } fn hasShardedAxis(self: Buffer) bool { diff --git a/zml/exe.zig b/zml/exe.zig index 946410c6..01bf0142 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -197,25 +197,16 @@ pub const BaseExe = struct { } pub fn _unsafeCall(self: BaseExe) void { - var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; - const sharding = self.platform.sharding(); - self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.input_buffer_count, .results = self.output_per_device, - .events = events[0..sharding.num_partitions], + .events = &.{}, // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, }) catch unreachable; - - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } } pub fn serialize(self: BaseExe, writer: anytype) !void { diff --git a/zml/module.zig b/zml/module.zig index 011e6fe6..fdfd2dd9 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -444,7 +444,7 @@ pub const CompilationContext = struct { attr.appendAssumeCapacity(.named( mlir_ctx, "mhlo.memory_kind", - .string(mlir_ctx, memory_kind.pjrtName()), + .string(mlir_ctx, @tagName(memory_kind.toPjrt())), )); } } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index ba0665a0..4eaaef87 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -4,6 +4,9 @@ const asynk = @import("async"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); +const stdx = @import("stdx"); +const tracer = @import("tools/tracer.zig"); + pub const ffi = pjrt.ffi; pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; @@ -20,6 +23,7 @@ pub const SerializeResult = pjrt.SerializeResult; pub const Executable = pjrt.Executable; pub const ExecuteError = ApiError; pub const Memory = pjrt.Memory; +pub const TopologyDescription = pjrt.TopologyDescription; const log = std.log.scoped(.zml); @@ -62,6 +66,13 @@ pub const Client = opaque { return .{ @ptrCast(buffer), @ptrCast(event_) }; } + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaMap(api, data); + } + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaUnmap(api, data); + } + pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { return @ptrCast(try asynk.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); } @@ -116,13 +127,13 @@ pub const Client = opaque { return self.inner().addressableMemories(api); } - pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*Memory { - for (self.addressableMemories(api)) |mem| { - if (mem.kind(api) == kind) { - return mem; - } - } - return null; + pub const CreateBuffersForAsyncHostToDeviceArgs = pjrt.Client.CreateBuffersForAsyncHostToDeviceArgs; + pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { + return self.inner().createBuffersForAsyncHostToDevice(api, args); + } + + pub fn topologyDescription(self: *const Client, api: *const Api) *const TopologyDescription { + return self.inner().topology_description(api); } }; @@ -149,6 +160,10 @@ pub const Buffer = opaque { return self.inner().isOnCpu(api); } + pub fn memory(self: *const Buffer, api: *const Api) *const Memory { + return self.inner().memory(api); + } + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { return @ptrCast(try self.inner().toHostBuffer(api, dst)); } @@ -173,6 +188,10 @@ pub const Buffer = opaque { return @ptrCast(self.inner().copyToDevice(api, device)); } + pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { + return @ptrCast(try self.inner().copyToMemory(api, memory_)); + } + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { return @ptrCast(self.inner().getReadyEvent(api)); } diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..c72ca280 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -127,7 +127,7 @@ const _CreateOptions = struct { pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { - values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); + values.appendAssumeCapacity(pjrt.NamedValue.from("allocator", "platform")); }, .bfc, .@"async" => |opt| { values.appendAssumeCapacity(pjrt.NamedValue.from("allocator", self.allocator)); @@ -136,7 +136,7 @@ const _CreateOptions = struct { values.appendAssumeCapacity(pjrt.NamedValue.from("memory_fraction", opt.memory_fraction)); } if (opt.collective_memory_size_mb > 0) { - const collective = @as(i64, opt.collective_memory_size_mb) * 1024 * 1024; + const collective = opt.collective_memory_size_mb * 1024 * 1024; values.appendAssumeCapacity(pjrt.NamedValue.from("collective_memory_size", collective)); } }, diff --git a/zml/shape.zig b/zml/shape.zig index 8da8519b..699bb1e7 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -17,6 +17,8 @@ test { /// Represent the shape of a tensor. pub const Shape = struct { pub const MAX_RANK: u8 = 8; + pub const MINOR_TO_MAJOR: [MAX_RANK]i64 = std.simd.reverseOrder(std.simd.iota(i64, MAX_RANK)); + pub const MAJOR_TO_MINOR: [MAX_RANK]i64 = std.simd.iota(i64, MAX_RANK); pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; @@ -1051,4 +1053,12 @@ pub const Shape = struct { } return res_shape; } + + pub fn minor_to_major(self: Shape) []const i64 { + return MINOR_TO_MAJOR[MAX_RANK - self.rank() ..]; + } + + pub fn major_to_minor(self: Shape) []const i64 { + return MAJOR_TO_MINOR[0..self.rank()]; + } }; diff --git a/zml/tensor.zig b/zml/tensor.zig index 79c2f2c9..7ec60c72 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -207,7 +207,7 @@ pub const Tensor = struct { const mlir_ctx = ctx.mlirCtx(); if (ctx.target() == .cpu) return self; - const memory_kind = @tagName(kind.toPjrtMemory()); + const memory_kind = @tagName(kind.toPjrt()); const frontend_attributes = mlir.Attribute.dict(mlir_ctx, &.{ .{ "_xla_buffer_placement", .string(mlir_ctx, memory_kind) },