From ac89abcf675e8b390a18c60a55a50b1971e4c191 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 7 Jan 2025 22:55:02 +0100 Subject: [PATCH 01/15] zml: make buffers and execute asynchronous This change makes buffer and execution completely asynchronous. This enable easy pipelining of multiple operations. It doesn't map well to `async` because the PJRT API is already asynchronous, so might as well use it as is. --- examples/mnist/mnist.zig | 4 +-- pjrt/pjrt.zig | 2 +- zml/buffer.zig | 44 +++++++++++++----------- zml/pjrtx.zig | 73 ++++++++++++++++++++++++++++++++++------ 4 files changed, 91 insertions(+), 32 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c49300a9..7b643db7 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -101,11 +101,11 @@ pub fn asyncMain() !void { const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); + var input = try zml.Buffer.from2(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); + var result: zml.Buffer = mnist.call(.{try input.awaitt()}); defer result.deinit(); log.info( diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index f48b6361..ab41f89c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -811,7 +811,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, diff --git a/zml/buffer.zig b/zml/buffer.zig index 23691382..66e1a8ea 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -79,38 +79,39 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - .{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - - frames.appendAssumeCapacity(frame); - } - - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } res._shards.appendAssumeCapacity(pjrt_buffer); } + return res; } + pub fn awaitt(self: Buffer) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.awaitt(self._api); + } + } + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -131,6 +132,11 @@ pub const Buffer = struct { return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); } + pub fn fromSlice2(platform: Platform, dimz: anytype, s: anytype) !Buffer { + const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); + return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); + } + /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index ae17f516..77975216 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -63,13 +63,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return @ptrCast(buffer); + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -196,6 +192,62 @@ pub const Buffer = opaque { } }; +pub const Buffer = opaque { + pub const inner = InnerMixin(pjrt.Buffer).inner; + + pub fn deinit(self: *Buffer, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { + return try self.inner().getDevice(api); + } + + pub fn delete(self: *Buffer, api: *const Api) void { + self.inner().delete(api); + } + + pub fn isDeleted(self: *const Buffer, api: *const Api) bool { + return self.inner().isDeleted(api); + } + + pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { + return self.inner().isOnCpu(api); + } + + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { + return @ptrCast(try self.inner().toHostBuffer(api, dst)); + } + + pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { + return self.inner().getElementType(api); + } + + pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { + return self.inner().getDimensions(api); + } + + pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { + return try self.inner().getUnpaddedDimensions(api); + } + + pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { + return try self.inner().getOnDeviceSizeInBytes(api); + } + + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { + return @ptrCast(self.inner().copyToDevice(api, device)); + } + + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { + return @ptrCast(self.inner().getReadyEvent(api)); + } + + pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { + return try self.inner().getOpaqueDeviceMemoryDataPointer(api); + } +}; + pub const Event = opaque { pub const inner = InnerMixin(pjrt.Event).inner; @@ -211,7 +263,8 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) !void { + pub const await_ = awaitt; + pub fn awaitt(self: *Event, api: *const Api) !void { defer self.deinit(api); if (self.isReady(api)) { @@ -267,13 +320,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From f6b08dd4cbb01b4d74d556347acd5be69b78ae86 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 20 Jan 2025 19:12:39 +0100 Subject: [PATCH 02/15] zml/aio: wait for buffer readiness in loadBuffers --- zml/aio.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/aio.zig b/zml/aio.zig index 175ffb32..840c84fe 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -592,7 +592,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from(platform, buf_with_metadata); + obj.* = try (try zml.Buffer.from(platform, buf_with_metadata)).awaitt(); } else { return error.BufferNotFound; }; From fa65c5b6e7a870e0a60eded5a79d0b04c21db493 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Mon, 20 Jan 2025 19:47:56 +0100 Subject: [PATCH 03/15] llama: fix async toHost usage when fetching token from device. --- examples/llama/main.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 87148371..5e9223c9 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -129,9 +129,11 @@ pub fn generateText( tracer.frameEnd(frame_id, try std.fmt.bufPrintZ(tracer_buffer, "Generated token {}/{}", .{ i + 1, output_tokens_len })); - // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + // extract the generated token from the buffer, async + var fut_tok_buf = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + _ = try fut_tok_buf.awaitt(); const generated_token = generated_token_buffer[0]; + // de-tokenize generated token into a string const chunk = try tokenizer_decoder.next(@intCast(generated_token)) orelse unreachable; num_tokens_generated = i; From bf273aa32f83a5bf750569443061f56906faf7ae Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 10:19:42 +0000 Subject: [PATCH 04/15] zml/aio: await weights transfers --- zml/aio.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zml/aio.zig b/zml/aio.zig index 840c84fe..26d3e73c 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -592,7 +592,8 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try (try zml.Buffer.from(platform, buf_with_metadata)).awaitt(); + obj.* = try zml.Buffer.from(platform, buf_with_metadata); + _ = try obj.awaitt(); } else { return error.BufferNotFound; }; From 864632fde3bbf67253249eb515e55aa3e0a3847b Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 10:20:09 +0000 Subject: [PATCH 05/15] zml: add Buffer.Shard type --- zml/buffer.zig | 49 +++++++++++++++++++++++++++---------------------- zml/exe.zig | 4 ++-- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 66e1a8ea..d5ec386a 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -55,7 +55,7 @@ pub const Buffer = struct { _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; - pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const Shards = std.BoundedArray(Shard, MAX_NUM_SHARDS); /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { @@ -94,20 +94,19 @@ pub const Buffer = struct { .device = devices[i], .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - if (event) |ev| { - ev.deinit(platform.pjrt_api); - } - res._shards.appendAssumeCapacity(pjrt_buffer); + res._shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + .ready_event = event, + }); } return res; } - pub fn awaitt(self: Buffer) !Buffer { - for (self._shards.constSlice()) |buffer| { - if (buffer.getReadyEvent(self._api)) |ev| { - try ev.awaitt(self._api); - } + pub fn awaitt(self: *Buffer) !*Buffer { + for (self._shards.slice()) |*shard| { + try shard.awaitt(); } return self; } @@ -117,7 +116,12 @@ pub const Buffer = struct { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); + for (pjrt_buffers) |pjrt_buffer| { + shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + }); + } return .{ ._api = platform.pjrt_api, ._shape = shape_, @@ -274,11 +278,12 @@ pub const Buffer = struct { /// The returned `HostBuffer` doesn't own the memory. pub fn toHost(self: Buffer, output: []u8) !HostBuffer { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, output); + var hb = HostBuffer.fromBytes(self.shape(), output); + hb._event = maybe_event; + hb._api = self._api; + hb._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return hb; } /// Copies the content of the Buffer to the host. @@ -286,10 +291,10 @@ pub const Buffer = struct { pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { const output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); - if (maybe_event) |event| { - try event.await_(self._api); - } + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, @constCast(output.data)); + output._event = maybe_event; + output._api = self._api; + output._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; return output; } @@ -297,8 +302,8 @@ pub const Buffer = struct { /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *const Buffer) void { - for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._api); + for (self._shards.constSlice()) |shard| { + shard.buffer.deinit(self._api); } } diff --git a/zml/exe.zig b/zml/exe.zig index b34d5e9b..5105184a 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -316,7 +316,7 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { const model_sharding = ctx.buffers.len; stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {}-sharded tensor into a {}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); for (buffer._shards.constSlice(), 0..) |shard, d| { - ctx.buffers[d][ctx.index] = shard; + ctx.buffers[d][ctx.index] = shard.buffer; } ctx.index += 1; } @@ -344,7 +344,7 @@ fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Bu ctx.index += 1; if (i >= ctx.buffer_shapes.len) return; - var shards: Buffer.Shards = .{}; + var shards: std.BoundedArray(*pjrt.Buffer, Buffer.MAX_NUM_SHARDS) = .{}; for (ctx.buffers) |buff| { shards.appendAssumeCapacity(buff[i]); } From b20c3d07e200d4c67e165a22e7135ba1fa018079 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 10:20:28 +0000 Subject: [PATCH 06/15] zml/exe: don't await for shard after execution --- zml/exe.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index 5105184a..b053f8bf 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -210,11 +210,11 @@ pub const BaseExe = struct { .non_donatable_input_indices = &.{}, }) catch unreachable; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn serialize(self: BaseExe, writer: anytype) !void { From 3d9b2aab649dc8651d2ccba7a129411fa2ed40c4 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 21 Jan 2025 12:15:27 +0100 Subject: [PATCH 07/15] zml(buffer): fix const bug in toHostAlloc --- zml/buffer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index d5ec386a..8d763bd6 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -289,7 +289,7 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); + var output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, @constCast(output.data)); output._event = maybe_event; From 1cef4fce04d7e6f1229fbb5ff153e5c68c9d3807 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 15:23:34 +0100 Subject: [PATCH 08/15] zml,mnist: fix async buffers handling --- examples/mnist/mnist.zig | 4 ++-- zml/buffer.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 7b643db7..c49300a9 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -101,11 +101,11 @@ pub fn asyncMain() !void { const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from2(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); + var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{try input.awaitt()}); + var result: zml.Buffer = mnist.call(.{input}); defer result.deinit(); log.info( diff --git a/zml/buffer.zig b/zml/buffer.zig index 8d763bd6..c668d2d8 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -266,7 +266,7 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { try event.await_(self._api); } From 260e3e10e3896bf43b9c0d12f069e6dabf14f698 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 20 Jan 2025 14:46:22 +0100 Subject: [PATCH 09/15] zml: add asynchronous HostBuffer event --- zml/buffer.zig | 2 +- zml/hostbuffer.zig | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index c668d2d8..dd6712ea 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -268,7 +268,7 @@ pub const Buffer = struct { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.awaitt(self._api); } return res; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index cd7e3a60..93daef2c 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -1,6 +1,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); + const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -22,6 +24,9 @@ pub const HostBuffer = struct { managed: u5, unmanaged, } = .unmanaged, + _ready: bool = true, + _api: ?*const pjrt.Api = null, + _event: ?*pjrt.Event = null, /// Allocates a HostBuffer with the given shape. /// The memory is left undefined. @@ -255,6 +260,20 @@ pub const HostBuffer = struct { }; } + pub fn awaitt(self: *HostBuffer) !*HostBuffer { + if (self._ready) { + return self; + } + if (self._event) |ev| { + if (self._api) |api| { + try ev.awaitt(api); + } else { + stdx.debug.panic("Got an event but no API. This should not happen.", .{}); + } + } + return self; + } + pub fn format( self: HostBuffer, comptime fmt: []const u8, From d679c7fed12054f23a679c75b1e565af4db270a8 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 26 Jan 2025 12:15:35 +0000 Subject: [PATCH 10/15] zml/buffer: await constant buffer creation --- zml/buffer.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index dd6712ea..1f94e502 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -201,7 +201,9 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, .data = &bytes }; - return try from(platform, host_buffer); + var buf = try from(platform, host_buffer); + _ = try buf.awaitt(); + return buf; } test constant { From 137eb86657de0727dcee92c1bf48d2989aa032ed Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 26 Jan 2025 12:16:04 +0000 Subject: [PATCH 11/15] zml/hostbuffer: is buffer is ready, mark it ready --- zml/hostbuffer.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 93daef2c..da96cff9 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -271,6 +271,7 @@ pub const HostBuffer = struct { stdx.debug.panic("Got an event but no API. This should not happen.", .{}); } } + self._ready = true; return self; } From 9f931db65dd3f95baf95dbdf78d8cd38f102e6d2 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 26 Jan 2025 12:16:32 +0000 Subject: [PATCH 12/15] zml/module: disable CUDA libnvptx for older plugin --- zml/module.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 0c1bc19f..a003896d 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -905,8 +905,8 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables setFlag(&options, "xla_gpu_enable_triton_gemm", false); setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); - setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); - setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); + // setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); + // setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); From b95577544447564e3204be172cb766a9237a51d3 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 31 Jan 2025 13:07:53 +0100 Subject: [PATCH 13/15] async: switch Mutex lock syntax --- async/async.zig | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/async/async.zig b/async/async.zig index 79026da2..1ce5148c 100644 --- a/async/async.zig +++ b/async/async.zig @@ -551,12 +551,9 @@ pub fn logFn(comptime fallbackLogFn: LogFn) LogFn { var bw = std.io.bufferedWriter(stderr); const writer = bw.writer(); - var mutex = Self.mu orelse blk: { - Self.mu = Mutex.init(); - break :blk Self.mu.?; - }; - mutex.lock(); - defer mutex.unlock(); + Self.mu = Self.mu orelse Mutex.init(); + Self.mu.?.lock(); + defer Self.mu.?.unlock(); nosuspend { writer.print(level_txt ++ prefix2 ++ format ++ "\n", args) catch return; bw.flush() catch return; From f974516cbcaff646a784b36fd60702c172408818 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 31 Jan 2025 13:08:20 +0100 Subject: [PATCH 14/15] pjrt: add PJRT_Memory bindings --- pjrt/pjrt.zig | 10 +++++++++ zml/pjrtx.zig | 58 +-------------------------------------------------- 2 files changed, 11 insertions(+), 57 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index ab41f89c..e80b97db 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -502,6 +502,16 @@ pub const Device = opaque { }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) []*const Memory { + const ret = api.call(.PJRT_Device_AddressableMemories, .{ + .device = self.inner(), + }) catch unreachable; + if (ret.memories) |memories| { + return @ptrCast(memories[0..ret.num_memories]); + } + return &.{}; + } }; pub const DeviceDescription = opaque { diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 77975216..ea59219a 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -28,8 +28,8 @@ pub const Error = pjrt.Error; pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; pub const SerializeResult = pjrt.SerializeResult; pub const Executable = pjrt.Executable; -pub const ExecuteError = ApiError; pub const Memory = pjrt.Memory; +pub const ExecuteError = ApiError; fn InnerMixin(comptime innerT: type) type { return struct { @@ -192,62 +192,6 @@ pub const Buffer = opaque { } }; -pub const Buffer = opaque { - pub const inner = InnerMixin(pjrt.Buffer).inner; - - pub fn deinit(self: *Buffer, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { - return try self.inner().getDevice(api); - } - - pub fn delete(self: *Buffer, api: *const Api) void { - self.inner().delete(api); - } - - pub fn isDeleted(self: *const Buffer, api: *const Api) bool { - return self.inner().isDeleted(api); - } - - pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { - return self.inner().isOnCpu(api); - } - - pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { - return @ptrCast(try self.inner().toHostBuffer(api, dst)); - } - - pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { - return self.inner().getElementType(api); - } - - pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { - return self.inner().getDimensions(api); - } - - pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { - return try self.inner().getUnpaddedDimensions(api); - } - - pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { - return try self.inner().getOnDeviceSizeInBytes(api); - } - - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { - return @ptrCast(self.inner().copyToDevice(api, device)); - } - - pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { - return @ptrCast(self.inner().getReadyEvent(api)); - } - - pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { - return try self.inner().getOpaqueDeviceMemoryDataPointer(api); - } -}; - pub const Event = opaque { pub const inner = InnerMixin(pjrt.Event).inner; From 2d1a99d4c311e80c6e7bee17894bbfb6124956a1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 11 Feb 2025 13:03:36 +0000 Subject: [PATCH 15/15] zml/exe.zig: added a way tu dupe the exe --- zml/exe.zig | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/zml/exe.zig b/zml/exe.zig index b053f8bf..ac4f1d41 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -190,6 +190,13 @@ pub const BaseExe = struct { self._arena.deinit(); } + pub fn dupe(self: BaseExe, allocator: std.mem.Allocator) !BaseExe { + var copy = try BaseExe.init(allocator, self.platform, self.exe, .{ .n_in = self.input_buffer_count, .result_shapes = self.result_shapes, .n_devices = self.num_devices }); + errdefer copy.deinit(); + + return copy; + } + pub fn call(self: BaseExe) void { stdx.debug.assert(self.input_buffer_count == self.ready_buffer_count, "BaseExe isn't ready to be called, expected {} buffer inputs got {}", .{ self.input_buffer_count, self.ready_buffer_count }); return self._unsafeCall(); @@ -270,6 +277,15 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + pub fn dupeAndPrepare(self: Self, allocator: std.mem.Allocator, first_arg: Bufferized(stdx.meta.Head(ArgsT))) !Exe(stdx.meta.Tail(ArgsT), ReturnT) { + var new_inner = try self.inner.dupe(allocator); + errdefer new_inner.deinit(); + + new_inner.prepare(first_arg); + + return .{ .inner = new_inner }; + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); }