From 7980fb51ff970096938c67beca3f0abc73c9cce0 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 7 Jan 2025 22:55:02 +0100 Subject: [PATCH 01/27] zml: make buffers and execute asynchronous This change makes buffer and execution completely asynchronous. This enable easy pipelining of multiple operations. It doesn't map well to `async` because the PJRT API is already asynchronous, so might as well use it as is. --- examples/llama/main.zig | 2 +- examples/mnist/mnist.zig | 4 +-- pjrt/pjrt.zig | 2 +- zml/buffer.zig | 44 +++++++++++++----------- zml/pjrtx.zig | 74 ++++++++++++++++++++++++++++++++++------ 5 files changed, 92 insertions(+), 34 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6c2a9df0..3215f6d6 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -166,7 +166,7 @@ const params = clap.parseParamsComptime( \\--seed random seed (optional) \\--seq-len sequence length \\--create-options platform creation options JSON, defaults to {} - \\--no-llama3 skip prompt template + \\--no-llama3 skip prompt template \\--sharding default: true: sharding on or off ); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c49300a9..7b643db7 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -101,11 +101,11 @@ pub fn asyncMain() !void { const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); + var input = try zml.Buffer.from2(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); + var result: zml.Buffer = mnist.call(.{try input.awaitt()}); defer result.deinit(); log.info( diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9cfa5fb5..651a1a55 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -732,7 +732,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, diff --git a/zml/buffer.zig b/zml/buffer.zig index a9fba67a..f149ef04 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -56,38 +56,39 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - .{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - - frames.appendAssumeCapacity(frame); - } - - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } res._shards.appendAssumeCapacity(pjrt_buffer); } + return res; } + pub fn awaitt(self: Buffer) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.awaitt(self._api); + } + } + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -108,6 +109,11 @@ pub const Buffer = struct { return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); } + pub fn fromSlice2(platform: Platform, dimz: anytype, s: anytype) !Buffer { + const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); + return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); + } + /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 7408e074..cf68dba0 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -17,7 +17,6 @@ const log = std.log.scoped(.zml); pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; -pub const Buffer = pjrt.Buffer; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; @@ -63,13 +62,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return buffer; + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -130,6 +125,62 @@ pub const Client = opaque { } }; +pub const Buffer = opaque { + pub const inner = InnerMixin(pjrt.Buffer).inner; + + pub fn deinit(self: *Buffer, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { + return try self.inner().getDevice(api); + } + + pub fn delete(self: *Buffer, api: *const Api) void { + self.inner().delete(api); + } + + pub fn isDeleted(self: *const Buffer, api: *const Api) bool { + return self.inner().isDeleted(api); + } + + pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { + return self.inner().isOnCpu(api); + } + + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { + return @ptrCast(try self.inner().toHostBuffer(api, dst)); + } + + pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { + return self.inner().getElementType(api); + } + + pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { + return self.inner().getDimensions(api); + } + + pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { + return try self.inner().getUnpaddedDimensions(api); + } + + pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { + return try self.inner().getOnDeviceSizeInBytes(api); + } + + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { + return @ptrCast(self.inner().copyToDevice(api, device)); + } + + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { + return @ptrCast(self.inner().getReadyEvent(api)); + } + + pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { + return try self.inner().getOpaqueDeviceMemoryDataPointer(api); + } +}; + pub const Event = opaque { pub const inner = InnerMixin(pjrt.Event).inner; @@ -145,7 +196,8 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) !void { + pub const await_ = awaitt; + pub fn awaitt(self: *Event, api: *const Api) !void { defer self.deinit(api); if (self.isReady(api)) { @@ -201,13 +253,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 8a5b73e70bb03d399eeba4f63bfd975967191c74 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 20 Jan 2025 19:12:39 +0100 Subject: [PATCH 02/27] zml/aio: wait for buffer readiness in loadBuffers --- zml/aio.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/aio.zig b/zml/aio.zig index 55155c48..2f20f4f6 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -589,7 +589,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from(platform, buf_with_metadata); + obj.* = try (try zml.Buffer.from(platform, buf_with_metadata)).awaitt(); } else { return error.BufferNotFound; }; From 9ca43972d85a153b0ba2ff952268bac2fa7f1c09 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Mon, 20 Jan 2025 19:47:56 +0100 Subject: [PATCH 03/27] llama: fix async toHost usage when fetching token from device. --- examples/llama/main.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 3215f6d6..2517fa19 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -128,9 +128,11 @@ pub fn generateText( tracer.frameEnd(frame_id, try std.fmt.bufPrintZ(tracer_buffer, "Generated token {}/{}", .{ i + 1, output_tokens_len })); - // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + // extract the generated token from the buffer, async + var fut_tok_buf = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + _ = try fut_tok_buf.awaitt(); const generated_token = generated_token_buffer[0]; + // de-tokenize generated token into a string const chunk = try tokenizer_decoder.next(@intCast(generated_token)) orelse unreachable; num_tokens_generated = i; From 9517504d02e82ad348d3c5471fc2068764a78e4c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 10:19:42 +0000 Subject: [PATCH 04/27] zml/aio: await weights transfers --- zml/aio.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zml/aio.zig b/zml/aio.zig index 2f20f4f6..90206259 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -589,7 +589,8 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try (try zml.Buffer.from(platform, buf_with_metadata)).awaitt(); + obj.* = try zml.Buffer.from(platform, buf_with_metadata); + _ = try obj.awaitt(); } else { return error.BufferNotFound; }; From 2612952a440268bb4945bd141dd23996ff3c5176 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 10:20:09 +0000 Subject: [PATCH 05/27] zml: add Buffer.Shard type --- zml/buffer.zig | 66 +++++++++++++++++++++++++++++++++----------------- zml/exe.zig | 4 +-- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index f149ef04..7d2afa1d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -27,12 +27,29 @@ const log = std.log.scoped(.zml); /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { + pub const Shard = struct { + api: *const pjrt.Api, + buffer: *pjrt.Buffer, + ready_event: ?*pjrt.Event = null, + ready: bool = false, + + pub fn awaitt(self: *Shard) !void { + if (self.ready) { + return; + } + if (self.ready_event orelse self.buffer.getReadyEvent(self.api)) |ev| { + try ev.awaitt(self.api); + } + self.ready = true; + } + }; + _shape: Shape, _api: *const pjrt.Api, _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; - pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const Shards = std.BoundedArray(Shard, MAX_NUM_SHARDS); /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { @@ -71,20 +88,19 @@ pub const Buffer = struct { .device = devices[i], .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - if (event) |ev| { - ev.deinit(platform.pjrt_api); - } - res._shards.appendAssumeCapacity(pjrt_buffer); + res._shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + .ready_event = event, + }); } return res; } - pub fn awaitt(self: Buffer) !Buffer { - for (self._shards.constSlice()) |buffer| { - if (buffer.getReadyEvent(self._api)) |ev| { - try ev.awaitt(self._api); - } + pub fn awaitt(self: *Buffer) !*Buffer { + for (self._shards.slice()) |*shard| { + try shard.awaitt(); } return self; } @@ -94,7 +110,12 @@ pub const Buffer = struct { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); + for (pjrt_buffers) |pjrt_buffer| { + shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + }); + } return .{ ._api = platform.pjrt_api, ._shape = shape_, @@ -251,11 +272,12 @@ pub const Buffer = struct { /// The returned `HostBuffer` doesn't own the memory. pub fn toHost(self: Buffer, output: []u8) !HostBuffer { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, output); + var hb = HostBuffer.fromBytes(self.shape(), output); + hb._event = maybe_event; + hb._api = self._api; + hb._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return hb; } /// Copies the content of the Buffer to the host. @@ -263,10 +285,10 @@ pub const Buffer = struct { pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { const output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); - if (maybe_event) |event| { - try event.await_(self._api); - } + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, @constCast(output.data)); + output._event = maybe_event; + output._api = self._api; + output._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; return output; } @@ -274,8 +296,8 @@ pub const Buffer = struct { /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *const Buffer) void { - for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._api); + for (self._shards.constSlice()) |shard| { + shard.buffer.deinit(self._api); } } diff --git a/zml/exe.zig b/zml/exe.zig index b34d5e9b..5105184a 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -316,7 +316,7 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { const model_sharding = ctx.buffers.len; stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {}-sharded tensor into a {}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); for (buffer._shards.constSlice(), 0..) |shard, d| { - ctx.buffers[d][ctx.index] = shard; + ctx.buffers[d][ctx.index] = shard.buffer; } ctx.index += 1; } @@ -344,7 +344,7 @@ fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Bu ctx.index += 1; if (i >= ctx.buffer_shapes.len) return; - var shards: Buffer.Shards = .{}; + var shards: std.BoundedArray(*pjrt.Buffer, Buffer.MAX_NUM_SHARDS) = .{}; for (ctx.buffers) |buff| { shards.appendAssumeCapacity(buff[i]); } From 85698b515e28ed2233016db38ffab3a4a61b1faf Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 10:20:28 +0000 Subject: [PATCH 06/27] zml/exe: don't await for shard after execution --- zml/exe.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index 5105184a..b053f8bf 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -210,11 +210,11 @@ pub const BaseExe = struct { .non_donatable_input_indices = &.{}, }) catch unreachable; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn serialize(self: BaseExe, writer: anytype) !void { From 9a683a87b28bf076d2448108fe35b897e6b26022 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 21 Jan 2025 12:15:27 +0100 Subject: [PATCH 07/27] zml(buffer): fix const bug in toHostAlloc --- zml/buffer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 7d2afa1d..43e8f7cc 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -283,7 +283,7 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); + var output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, @constCast(output.data)); output._event = maybe_event; From 1141fa7f9d40dab8f7557feaad29ef49abdc7543 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 15:23:34 +0100 Subject: [PATCH 08/27] zml,mnist: fix async buffers handling --- examples/mnist/mnist.zig | 4 ++-- zml/buffer.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 7b643db7..c49300a9 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -101,11 +101,11 @@ pub fn asyncMain() !void { const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from2(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); + var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{try input.awaitt()}); + var result: zml.Buffer = mnist.call(.{input}); defer result.deinit(); log.info( diff --git a/zml/buffer.zig b/zml/buffer.zig index 43e8f7cc..bacfd39e 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -260,7 +260,7 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { try event.await_(self._api); } From b3fdf197fed6f6d09ca6caeb42e53702ce14e8d5 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 20 Jan 2025 14:46:22 +0100 Subject: [PATCH 09/27] zml: add asynchronous HostBuffer event --- zml/buffer.zig | 2 +- zml/hostbuffer.zig | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index bacfd39e..eeac255c 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -262,7 +262,7 @@ pub const Buffer = struct { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.awaitt(self._api); } return res; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index cd7e3a60..93daef2c 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -1,6 +1,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); + const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -22,6 +24,9 @@ pub const HostBuffer = struct { managed: u5, unmanaged, } = .unmanaged, + _ready: bool = true, + _api: ?*const pjrt.Api = null, + _event: ?*pjrt.Event = null, /// Allocates a HostBuffer with the given shape. /// The memory is left undefined. @@ -255,6 +260,20 @@ pub const HostBuffer = struct { }; } + pub fn awaitt(self: *HostBuffer) !*HostBuffer { + if (self._ready) { + return self; + } + if (self._event) |ev| { + if (self._api) |api| { + try ev.awaitt(api); + } else { + stdx.debug.panic("Got an event but no API. This should not happen.", .{}); + } + } + return self; + } + pub fn format( self: HostBuffer, comptime fmt: []const u8, From 39a64d1041ccc0c70caf249319581a896111618a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 26 Jan 2025 12:15:35 +0000 Subject: [PATCH 10/27] zml/buffer: await constant buffer creation --- zml/buffer.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index eeac255c..86fa7f8a 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -195,7 +195,9 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, .data = &bytes }; - return try from(platform, host_buffer); + var buf = try from(platform, host_buffer); + _ = try buf.awaitt(); + return buf; } test constant { From 6e3879546bc1bf848afff9ede21cd927b89ceb0b Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 26 Jan 2025 12:16:04 +0000 Subject: [PATCH 11/27] zml/hostbuffer: is buffer is ready, mark it ready --- zml/hostbuffer.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 93daef2c..da96cff9 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -271,6 +271,7 @@ pub const HostBuffer = struct { stdx.debug.panic("Got an event but no API. This should not happen.", .{}); } } + self._ready = true; return self; } From f4eb7594b420ad798ce3c0c3ab3d2d24ed67c8df Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 26 Jan 2025 12:16:32 +0000 Subject: [PATCH 12/27] zml/module: disable CUDA libnvptx for older plugin --- zml/module.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 0c1bc19f..a003896d 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -905,8 +905,8 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables setFlag(&options, "xla_gpu_enable_triton_gemm", false); setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); - setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); - setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); + // setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); + // setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); From d9f2fa8d192d379d0174cca80890b03399ea97c1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 27 Jan 2025 17:56:31 +0000 Subject: [PATCH 13/27] zml/tensor.zig: add maximum as scatter operation --- zml/tensor.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zml/tensor.zig b/zml/tensor.zig index 23ab003b..f2e37497 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2419,6 +2419,10 @@ pub const Tensor = struct { /// of the operator which is backend specific. update_fn: *const fn (Tensor, Tensor) Tensor = increment, + pub fn maximum(old_value: Tensor, new_value: Tensor) Tensor { + return old_value.maximum(new_value.convert(old_value.dtype())); + } + pub fn increment(old_value: Tensor, new_value: Tensor) Tensor { return old_value.add(new_value.convert(old_value.dtype())); } From 01c54e0e72c8ed495a66deaf758e0fe84745f5e3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 29 Jan 2025 10:04:12 +0100 Subject: [PATCH 14/27] zml: exe should collect events --- zml/buffer.zig | 29 +++++++----------- zml/exe.zig | 80 ++++++++++++++++++++++++------------------------- zml/testing.zig | 2 ++ 3 files changed, 51 insertions(+), 60 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 86fa7f8a..bb343d7d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -28,6 +28,8 @@ const log = std.log.scoped(.zml); /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { pub const Shard = struct { + // TODO: remove api from here + // Also we should use SoA instead of AoS because of the "ready" bool api: *const pjrt.Api, buffer: *pjrt.Buffer, ready_event: ?*pjrt.Event = null, @@ -105,24 +107,6 @@ pub const Buffer = struct { return self; } - /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { - stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); - stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); - var shards: Shards = .{}; - for (pjrt_buffers) |pjrt_buffer| { - shards.appendAssumeCapacity(.{ - .api = platform.pjrt_api, - .buffer = pjrt_buffer, - }); - } - return .{ - ._api = platform.pjrt_api, - ._shape = shape_, - ._shards = shards, - }; - } - /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromSlice(platform: Platform, dimz: anytype, s: anytype) !Buffer { @@ -222,6 +206,7 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. + /// The returned buffer will be considered as ready to read. pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const anyopaque, device_data: *anyopaque) !Buffer { const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { var res: [Shape.MAX_RANK]i64 = undefined; @@ -249,7 +234,13 @@ pub const Buffer = struct { }); var shards: Shards = .{}; - shards.appendAssumeCapacity(pjrt_buffer); + shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + // We don't have event here so we assume it's ready + .ready_event = null, + .ready = true, + }); return .{ ._api = platform.pjrt_api, ._shape = shape_, diff --git a/zml/exe.zig b/zml/exe.zig index b053f8bf..6a5a226b 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -140,6 +140,8 @@ pub const BaseExe = struct { /// Pre-allocated slice of buffers to use as outputs when the module is called. output_per_device: []const [*]*pjrt.Buffer, + events_per_device: []?*pjrt.Event, + /// Number of buffers already fed to the executable. ready_buffer_count: u32, @@ -161,8 +163,8 @@ pub const BaseExe = struct { const n_out = args.result_shapes.len; const n_devices = args.n_devices; // Allocate once for all the *pjrt.Buffer we need to store ... - const all_buffers = try allocator.alloc(*pjrt.Buffer, (args.n_in + n_out) * n_devices); - const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ args.n_in * n_devices, n_out * n_devices }); + const all_buffers = try allocator.alloc(*pjrt.Buffer, (args.n_in + n_out + 1) * n_devices); + const all_input_buffers, const all_output_buffers, const events_per_device = splitBuffer(*pjrt.Buffer, all_buffers, .{ args.n_in * n_devices, n_out * n_devices, n_devices }); // ... and once for all the [*]*pjrt.Buffer. const all_per_device = try allocator.alloc([*]*pjrt.Buffer, 2 * n_devices); @@ -181,6 +183,7 @@ pub const BaseExe = struct { .num_devices = args.n_devices, .input_per_device = input_per_device, .output_per_device = output_per_device, + .events_per_device = @ptrCast(events_per_device), .result_shapes = try allocator.dupe(Shape, args.result_shapes), ._arena = arena, }; @@ -196,25 +199,16 @@ pub const BaseExe = struct { } pub fn _unsafeCall(self: BaseExe) void { - var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; - const sharding = self.platform.sharding(); - self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.input_buffer_count, .results = self.output_per_device, - .events = events[0..sharding.num_partitions], + .events = self.events_per_device, // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, }) catch unreachable; - - // for (events[0..sharding.num_partitions]) |e| { - // if (e) |ev| { - // ev.await_(self.platform.pjrt_api) catch unreachable; - // } - // } } pub fn serialize(self: BaseExe, writer: anytype) !void { @@ -235,13 +229,28 @@ pub const BaseExe = struct { self.ready_buffer_count += n; } - pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { - var shards: Buffer.Shards = .{}; - for (self.output_per_device) |dev_out| { - shards.appendAssumeCapacity(dev_out[i]); - } + pub fn numOutputs(self: BaseExe) u32 { + return @intCast(self.result_shapes.len); + } - return Buffer.fromPjrtBuffers(self.platform, self.result_shapes[i], shards.constSlice()); + /// Takes pjrt.Buffer from each device and make it into a Buffer. + /// Set the event of the buffer to the event of the call. + pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { + std.debug.assert(i < self.numOutputs()); + var shards: Buffer.Shards = .{}; + for (0..self.num_devices) |dev| { + shards.appendAssumeCapacity(.{ + .api = self.platform.pjrt_api, + .buffer = self.output_per_device[dev][i], + .ready_event = self.events_per_device[dev], + .ready = false, + }); + } + return .{ + ._shape = self.result_shapes[i], + ._api = self.platform.pjrt_api, + ._shards = shards, + }; } }; @@ -283,7 +292,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { std.debug.assert(total_ready == self.inner.input_buffer_count); self.inner._unsafeCall(); var result: Bufferized(ReturnT) = undefined; - assignRawBuffers(&result, self.inner.platform, self.inner.output_per_device, self.inner.result_shapes); + assignRawBuffers(&result, self.inner); return result; } }; @@ -325,31 +334,20 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { } /// Visit the given struct and override tensors by creating a new one using the provided PJRT buffers. -fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Buffer, buffer_shapes: []Shape) void { +fn assignRawBuffers(v: anytype, exe: BaseExe) void { const LocalContext = struct { - index: u32, - platform: Platform, - buffers: []const [*]*pjrt.Buffer, - buffer_shapes: []Shape, - }; - var local_ctx: LocalContext = .{ - .index = 0, - .platform = platform, - .buffers = buffers, - .buffer_shapes = buffer_shapes, - }; - meta.visit((struct { - fn cb(ctx: *LocalContext, buffer: *Buffer) void { + index: u32 = 0, + exe: *const BaseExe, + + fn cb(ctx: *@This(), buffer: *Buffer) void { const i = ctx.index; ctx.index += 1; - if (i >= ctx.buffer_shapes.len) return; + if (i >= ctx.exe.result_shapes.len) return; - var shards: std.BoundedArray(*pjrt.Buffer, Buffer.MAX_NUM_SHARDS) = .{}; - for (ctx.buffers) |buff| { - shards.appendAssumeCapacity(buff[i]); - } - buffer.* = Buffer.fromPjrtBuffers(ctx.platform, ctx.buffer_shapes[i], shards.constSlice()); + buffer.* = ctx.exe.getOutputBuffer(i); } - }).cb, &local_ctx, v); - stdx.debug.internalAssert(local_ctx.index == buffer_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ buffers.len, @typeName(@TypeOf(v)), local_ctx.index }); + }; + var local_ctx: LocalContext = .{ .exe = &exe }; + meta.visit(LocalContext.cb, &local_ctx, v); + stdx.debug.internalAssert(local_ctx.index == exe.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ exe.result_shapes.len, @typeName(@TypeOf(v)), local_ctx.index }); } diff --git a/zml/testing.zig b/zml/testing.zig index e686b603..4ae4140c 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -41,11 +41,13 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { .{ try left_.toHostAlloc(allocator), true } else .{ left_, false }; + _ = try left.awaitt(); var right: zml.HostBuffer, const should_free_right = if (@TypeOf(right_) == zml.Buffer) .{ try right_.toHostAlloc(allocator), true } else .{ right_, false }; + _ = try right.awaitt(); defer { if (should_free_left) left.deinit(allocator); From 6050d4f032a144cde20b9f3322d152c08564c56c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 29 Jan 2025 10:27:16 +0100 Subject: [PATCH 15/27] zml: remove api from Shard --- zml/buffer.zig | 15 ++++++--------- zml/exe.zig | 1 - 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index bb343d7d..71052895 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -28,19 +28,18 @@ const log = std.log.scoped(.zml); /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { pub const Shard = struct { - // TODO: remove api from here - // Also we should use SoA instead of AoS because of the "ready" bool - api: *const pjrt.Api, + // TODO: we should use SoA instead of AoS because of the "ready" bool. + // the Buffer struct is already huge. buffer: *pjrt.Buffer, ready_event: ?*pjrt.Event = null, ready: bool = false, - pub fn awaitt(self: *Shard) !void { + pub fn awaitt(self: *Shard, api: *const pjrt.Api) !void { if (self.ready) { return; } - if (self.ready_event orelse self.buffer.getReadyEvent(self.api)) |ev| { - try ev.awaitt(self.api); + if (self.ready_event orelse self.buffer.getReadyEvent(api)) |ev| { + try ev.awaitt(api); } self.ready = true; } @@ -91,7 +90,6 @@ pub const Buffer = struct { .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); res._shards.appendAssumeCapacity(.{ - .api = platform.pjrt_api, .buffer = pjrt_buffer, .ready_event = event, }); @@ -102,7 +100,7 @@ pub const Buffer = struct { pub fn awaitt(self: *Buffer) !*Buffer { for (self._shards.slice()) |*shard| { - try shard.awaitt(); + try shard.awaitt(self._api); } return self; } @@ -235,7 +233,6 @@ pub const Buffer = struct { var shards: Shards = .{}; shards.appendAssumeCapacity(.{ - .api = platform.pjrt_api, .buffer = pjrt_buffer, // We don't have event here so we assume it's ready .ready_event = null, diff --git a/zml/exe.zig b/zml/exe.zig index 6a5a226b..3c64f7f9 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -240,7 +240,6 @@ pub const BaseExe = struct { var shards: Buffer.Shards = .{}; for (0..self.num_devices) |dev| { shards.appendAssumeCapacity(.{ - .api = self.platform.pjrt_api, .buffer = self.output_per_device[dev][i], .ready_event = self.events_per_device[dev], .ready = false, From bc514c191d59786dc572dea276e9543ba58d8b55 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 30 Jan 2025 18:00:13 +0100 Subject: [PATCH 16/27] revert zml: exe should collect events --- zml/exe.zig | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index 3c64f7f9..db87ddf7 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -140,8 +140,6 @@ pub const BaseExe = struct { /// Pre-allocated slice of buffers to use as outputs when the module is called. output_per_device: []const [*]*pjrt.Buffer, - events_per_device: []?*pjrt.Event, - /// Number of buffers already fed to the executable. ready_buffer_count: u32, @@ -163,8 +161,8 @@ pub const BaseExe = struct { const n_out = args.result_shapes.len; const n_devices = args.n_devices; // Allocate once for all the *pjrt.Buffer we need to store ... - const all_buffers = try allocator.alloc(*pjrt.Buffer, (args.n_in + n_out + 1) * n_devices); - const all_input_buffers, const all_output_buffers, const events_per_device = splitBuffer(*pjrt.Buffer, all_buffers, .{ args.n_in * n_devices, n_out * n_devices, n_devices }); + const all_buffers = try allocator.alloc(*pjrt.Buffer, (args.n_in + n_out) * n_devices); + const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ args.n_in * n_devices, n_out * n_devices }); // ... and once for all the [*]*pjrt.Buffer. const all_per_device = try allocator.alloc([*]*pjrt.Buffer, 2 * n_devices); @@ -183,7 +181,6 @@ pub const BaseExe = struct { .num_devices = args.n_devices, .input_per_device = input_per_device, .output_per_device = output_per_device, - .events_per_device = @ptrCast(events_per_device), .result_shapes = try allocator.dupe(Shape, args.result_shapes), ._arena = arena, }; @@ -199,11 +196,15 @@ pub const BaseExe = struct { } pub fn _unsafeCall(self: BaseExe) void { + var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; + const sharding = self.platform.sharding(); + self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.input_buffer_count, .results = self.output_per_device, - .events = self.events_per_device, + // Create a local inste + .events = events[0..sharding.num_partitions], // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? @@ -241,7 +242,7 @@ pub const BaseExe = struct { for (0..self.num_devices) |dev| { shards.appendAssumeCapacity(.{ .buffer = self.output_per_device[dev][i], - .ready_event = self.events_per_device[dev], + .ready_event = null, .ready = false, }); } From 87b572226192258a946a3d21c4806795f75dc742 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 3 Feb 2025 14:24:35 +0100 Subject: [PATCH 17/27] zml.buffer.awaitAll --- zml/aio.zig | 3 +-- zml/buffer.zig | 28 ++++++++++++++++++++++++++-- zml/pjrtx.zig | 2 +- zml/zml.zig | 1 + 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/zml/aio.zig b/zml/aio.zig index 90206259..07f7c2a9 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -561,8 +561,7 @@ pub fn loadModelBuffersWithPrefix( return res; } -/// Takes a bufferized version of a `model`, ie a mirror struct of the `model`, and deinit all the -/// Buffer found. +/// deprecated, use zml.buffers.deinitAll() pub fn unloadBuffers(model: anytype) void { zml.meta.visit((struct { fn cb(_: void, buffer: *zml.Buffer) void { diff --git a/zml/buffer.zig b/zml/buffer.zig index 71052895..5dac9970 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -34,7 +34,7 @@ pub const Buffer = struct { ready_event: ?*pjrt.Event = null, ready: bool = false, - pub fn awaitt(self: *Shard, api: *const pjrt.Api) !void { + pub fn awaitt(self: *Shard, api: *const pjrt.Api) pjrt.ApiError!void { if (self.ready) { return; } @@ -98,7 +98,7 @@ pub const Buffer = struct { return res; } - pub fn awaitt(self: *Buffer) !*Buffer { + pub fn awaitt(self: *Buffer) pjrt.ApiError!*Buffer { for (self._shards.slice()) |*shard| { try shard.awaitt(self._api); } @@ -404,3 +404,27 @@ test bufferTypeFromDtype { try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); } } + +/// await all buffers in the given struct +pub fn awaitAll(buffers: anytype) pjrt.ApiError!void { + var err: ?pjrt.ApiError = null; + meta.visit((struct { + fn cb(err_ptr: *?pjrt.ApiError, buffer: *Buffer) void { + _ = buffer.awaitt() catch |e| { + err_ptr.* = e; + return; + }; + } + }).cb, &err, buffers); + + return err orelse {}; +} + +/// deinit all buffers in the given struct +pub fn deinitAll(buffers: anytype) void { + meta.visit((struct { + fn cb(_: void, buffer: *Buffer) void { + buffer.deinit(); + } + }).cb, {}, buffers); +} diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index cf68dba0..ae39af8f 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -197,7 +197,7 @@ pub const Event = opaque { } pub const await_ = awaitt; - pub fn awaitt(self: *Event, api: *const Api) !void { + pub fn awaitt(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); if (self.isReady(api)) { diff --git a/zml/zml.zig b/zml/zml.zig index 088ad687..8c075bcf 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -18,6 +18,7 @@ pub const Tensor = @import("tensor.zig").Tensor; // Namespaces pub const context = @import("context.zig"); +pub const buffer = @import("buffer.zig"); pub const exe = @import("exe.zig"); pub const floats = @import("floats.zig"); pub const helpers = @import("helpers.zig"); From 90903b78162d93bde0db8eec9e7ce53989a8d028 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 3 Feb 2025 16:53:50 +0100 Subject: [PATCH 18/27] fix fromArray --- zml/buffer.zig | 26 ++++++++++++++++++++------ zml/hostbuffer.zig | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 5dac9970..9bb6237b 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -112,16 +112,30 @@ pub const Buffer = struct { return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); } - pub fn fromSlice2(platform: Platform, dimz: anytype, s: anytype) !Buffer { - const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); - return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); - } - /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { const host_buffer = HostBuffer.fromArray(&arr); - return try from(platform, host_buffer); + var device_buffer = try from(platform, host_buffer); + // Await cause we need to finish the copy while the array is still on the stack. + // Alternatively we could accept a pointer to the array but that will break all fromArray callers + _ = try device_buffer.awaitt(); + return device_buffer; + } + + test fromArray { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const values_h = [2][8]i32{ + .{ 0, -1, 2, -3, 4, -5, 6, -7 }, + .{ 8, -9, 10, -11, 12, -13, 14, -15 }, + }; + var values_d = try fromArray(platform, values_h); + defer values_d.deinit(); + _ = try values_d.awaitt(); + + try std.testing.expectEqual(-15, (try values_d.getValue([2][8]i32))[1][7]); } /// Creates a Buffer with a single element. diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index da96cff9..d101c55f 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -103,6 +103,25 @@ pub const HostBuffer = struct { }; } + test fromArray { + const zml = @import("zml.zig"); + { + const arr: [2][4]i32 = undefined; + const buf = fromArray(&arr); + try zml.testing.expectEqualShapes(buf.shape(), Shape.init(.{ 2, 4 }, .i32)); + } + { + const arr: [2][4][8]u8 = undefined; + const buf = fromArray(&arr); + try zml.testing.expectEqualShapes(buf.shape(), Shape.init(.{ 2, 4, 8 }, .u8)); + } + { + const arr: [3][2][4]f32 = undefined; + const buf = fromArray(&arr); + try zml.testing.expectEqualShapes(buf.shape(), Shape.init(.{ 3, 2, 4 }, .f32)); + } + } + pub const ArangeArgs = struct { start: i64 = 0, end: i64, From 1c9ec996ee06dcecbcbee38675f321b08a6f1352 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 3 Feb 2025 16:54:10 +0100 Subject: [PATCH 19/27] fixup! bazel: use prebuilt protoc binaries (#178) --- MODULE.bazel.lock | 3 +++ examples/MODULE.bazel.lock | 3 +++ 2 files changed, 6 insertions(+) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index b23d9054..be285fcc 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -155,6 +155,7 @@ "https://bazel-registry.zml.ai/modules/stardoc/0.6.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.7.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.7.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/toolchains_protoc/0.3.7/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.8.0/MODULE.bazel": "not found", @@ -354,6 +355,8 @@ "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", "https://bcr.bazel.build/modules/stardoc/0.7.1/source.json": "b6500ffcd7b48cd72c29bb67bcac781e12701cc0d6d55d266a652583cfcdab01", + "https://bcr.bazel.build/modules/toolchains_protoc/0.3.7/MODULE.bazel": "d2758e5af5838970175274c4fcb67edc8d2f4f8f348885da90c42fafc221236d", + "https://bcr.bazel.build/modules/toolchains_protoc/0.3.7/source.json": "07e0eab9be4aded2a8074504a80c55c5a48e811a9824061642af639baaabf985", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", "https://bcr.bazel.build/modules/with_cfg.bzl/0.8.0/MODULE.bazel": "d836870543677303f1cb6f9649e955c772e9f15311c2926a2468c9a10eb214d9", diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 0ba88287..e97baa93 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -158,6 +158,7 @@ "https://bazel-registry.zml.ai/modules/stardoc/0.6.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.7.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.7.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/toolchains_protoc/0.3.7/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.8.0/MODULE.bazel": "not found", @@ -361,6 +362,8 @@ "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", "https://bcr.bazel.build/modules/stardoc/0.7.1/source.json": "b6500ffcd7b48cd72c29bb67bcac781e12701cc0d6d55d266a652583cfcdab01", + "https://bcr.bazel.build/modules/toolchains_protoc/0.3.7/MODULE.bazel": "d2758e5af5838970175274c4fcb67edc8d2f4f8f348885da90c42fafc221236d", + "https://bcr.bazel.build/modules/toolchains_protoc/0.3.7/source.json": "07e0eab9be4aded2a8074504a80c55c5a48e811a9824061642af639baaabf985", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", "https://bcr.bazel.build/modules/with_cfg.bzl/0.8.0/MODULE.bazel": "d836870543677303f1cb6f9649e955c772e9f15311c2926a2468c9a10eb214d9", From 6db444bf9a698df8bd8d22736fb2f1e2f098b90e Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 3 Feb 2025 17:05:59 +0100 Subject: [PATCH 20/27] fix partialSoftmax --- zml/nn.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index 6f756e99..7d10a3a5 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -934,7 +934,7 @@ pub const PartialSoftmax = struct { /// Returns intermediary results to allow aggregating later. pub fn partialSoftmax(self: Tensor, axis: anytype) PartialSoftmax { const a = self.axis(axis); - const max_val = self.max(a); + const max_val = self.max(a).maximum(Tensor.scalar(-1e16, self.dtype())); const out = self.sub(max_val.broad(self.shape())).exp(); return .{ .values = out, From 7baf092501e3b4c3f74b6aa4def4e4666b7d773a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 4 Feb 2025 12:14:42 +0100 Subject: [PATCH 21/27] examples/llama: read the token generated in prefill --- examples/llama/llama.zig | 9 ++-- examples/llama/main.zig | 110 ++++++++++++++++++--------------------- 2 files changed, 55 insertions(+), 64 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index a162b432..2eaf8a8b 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -87,16 +87,15 @@ pub const LlamaLM = struct { rng: Tensor.Rng, ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); - var tokens = tokens_.withPartialTags(.{.s}); + const tokens = tokens_.withPartialTags(.{.s}); const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); - tokens, const new_rng = self.sampleTokens(self.lm_head, tokens, out, rng, self.gen_opts); - return .{ tokens, updated_kv_cache, new_rng }; + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } pub fn sampleTokens( self: LlamaLM, lm_head_: ?zml.nn.Linear, - tokens_: Tensor, out_: Tensor, rng: Tensor.Rng, opts: zml.nn.SamplingStrategy, @@ -115,7 +114,7 @@ pub const LlamaLM = struct { logits = logits.rename(.{ .d = .voc }); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); - return .{ next_tokens.convert(tokens_.dtype()).reuseBuffer(tokens_), new_rng }; + return .{ next_tokens, new_rng }; } pub fn increment(_: u8, token_index: Tensor) Tensor { diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 87148371..7798a318 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -22,11 +22,17 @@ pub const std_options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; -pub fn tokenizePromptLlama3(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8) ![]u32 { +pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { var tokens = std.ArrayList(u32).init(allocator); var encoder = try tokenizer.encoder(); defer encoder.deinit(); + if (skip_llama3_encoding) { + // Copy to the arraylist so the ownership is the same in both branches. + try tokens.appendSlice(try encoder.encode(prompt)); + return tokens.toOwnedSlice(); + } + const start_header_id = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; const end_header_id = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; const eot_id = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; @@ -59,84 +65,63 @@ pub fn generateText( prompt: []const u8, skip_llama3_encoding: bool, ) ![]const u8 { - var tokenizer_encoder = try tokenizer.encoder(); - defer tokenizer_encoder.deinit(); + const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); + defer allocator.free(prompt_tok); + var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); - const prompt_tok: []const u32 = if (skip_llama3_encoding) try tokenizer_encoder.encode(prompt) else try tokenizePromptLlama3(allocator, tokenizer, config, prompt); - defer allocator.free(prompt_tok); - - const dims = llama_.model.shape(); - const max_seq_len = dims.s; - - // Prefill - // initialize a 0..max_seq_len buffer with the tokenized prompt - const prefill_buffer = try allocator.alloc(u32, @intCast(max_seq_len)); - @memset(prefill_buffer, 0); - for (0..prompt_tok.len) |i| { - prefill_buffer[i] = @intCast(prompt_tok[i]); - } - defer allocator.free(prefill_buffer); - const platform = mod_generate.platform(); + const max_seq_len = llama_.model.shape().s; - // prepare device buffers for the prefill tokens and the index - var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); - defer prefill_tokens.deinit(); - var prefill_token_index = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0); - - defer prefill_token_index.deinit(); - - // init RNG and prefill + // init RNG and buffers var rng = try zml.Tensor.Rng.init(platform, seed); - prefill_tokens, var kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_index, kv_cache_, rng }); + var generated_token_buffer = [_]u32{undefined}; + + var kv_cache = prefill: { + // prepare device buffers for the prefill tokens and their positions + const prefill_buffer = try allocator.alloc(u32, max_seq_len); + @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); + + var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); + defer prefill_tokens.deinit(); + var prefill_token_pos = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0); + defer prefill_token_pos.deinit(); + + const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); + _ = try prefilled_tokens.toHost(std.mem.sliceAsBytes(prefill_buffer)); + generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; + break :prefill kv_cache; + }; defer kv_cache.k.deinit(); defer kv_cache.v.deinit(); defer kv_cache.layer_index.deinit(); - // Prepare for token-by-token generation - var first_token_hostbuffer = [_]u32{prompt_tok[prompt_tok.len - 1]}; // start with the prompt's last token - var current_token = try zml.Buffer.fromSlice(platform, .{1}, &first_token_hostbuffer); + // Prepare for token-by-token generation, + // start with the token generated based on the full prompt. + var current_token = try zml.Buffer.fromSlice(platform, .{1}, &generated_token_buffer); defer current_token.deinit(); - // Here we will copy the generated token from device - var generated_token_buffer = [_]u32{0}; - // Here we collect the generated text var output = std.ArrayList(u8).init(allocator); defer output.deinit(); - const tracer_buffer = try allocator.alloc(u8, @intCast(max_seq_len)); - defer allocator.free(tracer_buffer); - const tracer = zml.tools.Tracer.init("ai.zml.models.llama"); const output_tokens_len = max_seq_len - prompt_tok.len - 1; const start = std.time.microTimestamp(); - var num_tokens_generated: usize = 0; + // One token has alreadyh been generated by the prefill. + var num_tokens_generated: usize = 1; - generation: for (0..output_tokens_len) |i| { - const frame_id = tracer.frameStart(try std.fmt.bufPrintZ(tracer_buffer, "Generate token {}/{}", .{ i + 1, output_tokens_len })); - - // current token index needs to go into a zml.Buffer - const token_index_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; - const token_index = try zml.Buffer.fromSlice(platform, .{}, token_index_buffer); - - defer token_index.deinit(); - - // call to generate the next token - current_token, kv_cache, rng = mod_generate.call(.{ current_token, token_index, kv_cache, rng }); - - tracer.frameEnd(frame_id, try std.fmt.bufPrintZ(tracer_buffer, "Generated token {}/{}", .{ i + 1, output_tokens_len })); - - // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + generation: for (0..output_tokens_len + 1) |i| { + // collect and print generated sequence + num_tokens_generated += 1; const generated_token = generated_token_buffer[0]; - // de-tokenize generated token into a string - const chunk = try tokenizer_decoder.next(@intCast(generated_token)) orelse unreachable; - num_tokens_generated = i; + const chunk = try tokenizer_decoder.next(generated_token) orelse unreachable; + try output.appendSlice(chunk); + std.debug.print("{s}", .{chunk}); // check for eos + if (i == output_tokens_len) break :generation; switch (config.eos_token_id.value) { .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, .ints => |eos_list| { @@ -146,9 +131,16 @@ pub fn generateText( }, } - // collect and print generated sequence - try output.appendSlice(chunk); - std.debug.print("{s}", .{chunk}); + // current token pos needs to go into a zml.Buffer + const token_pos_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; + const token_pos = try zml.Buffer.fromSlice(platform, .{}, token_pos_buffer); + defer token_pos.deinit(); + + // call to generate the next token + current_token, kv_cache, rng = mod_generate.call(.{ current_token, token_pos, kv_cache, rng }); + + // extract the generated token from the buffer + _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); } const end = std.time.microTimestamp(); const duration = stdx.math.divFloat(f64, end - start, std.time.us_per_s); From 576130b75039a75dae59fba95413c7910aea5683 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 27 Jan 2025 18:56:31 +0100 Subject: [PATCH 22/27] awaitAll --- examples/llama/main.zig | 4 +--- zml/aio.zig | 6 ++++++ zml/pjrtx.zig | 4 ++-- zml/tensor.zig | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 7798a318..9a17c404 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -93,9 +93,7 @@ pub fn generateText( generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; break :prefill kv_cache; }; - defer kv_cache.k.deinit(); - defer kv_cache.v.deinit(); - defer kv_cache.layer_index.deinit(); + defer zml.aio.unloadBuffers(&kv_cache); // Prepare for token-by-token generation, // start with the token generated based on the full prompt. diff --git a/zml/aio.zig b/zml/aio.zig index 175ffb32..27abbd8a 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -574,6 +574,12 @@ pub fn unloadBuffers(model: anytype) void { }).cb, {}, model); } +/// deinit all buffers in the given struct +pub fn awaitAll(buffers: anytype) !void { + // TODO: implement once we have async buffers. + _ = buffers; +} + fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, buffer_store: BufferStore, obj: anytype, platform: zml.Platform) !void { const err_msg = "visitStructAndLoadBuffer must be called with a pointer to type. Received "; const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 7408e074..a3f7705e 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -63,7 +63,7 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !*Buffer { + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!*Buffer { const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); if (event_) |event__| { const event: *Event = @ptrCast(event__); @@ -145,7 +145,7 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) !void { + pub fn await_(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); if (self.isReady(api)) { diff --git a/zml/tensor.zig b/zml/tensor.zig index 23ab003b..5529b6bc 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2420,11 +2420,12 @@ pub const Tensor = struct { update_fn: *const fn (Tensor, Tensor) Tensor = increment, pub fn increment(old_value: Tensor, new_value: Tensor) Tensor { - return old_value.add(new_value.convert(old_value.dtype())); + return old_value.add(new_value); } pub fn override(old_value: Tensor, new_value: Tensor) Tensor { - return new_value.convert(old_value.dtype()); + _ = old_value; + return new_value; } }; From 44874b3b7a8c5cedc6f8e888855807ce94e0eaa6 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 4 Feb 2025 16:45:46 +0100 Subject: [PATCH 23/27] --xla_dump_hlo_as_proto --- zml/module.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/module.zig b/zml/module.zig index 0c1bc19f..253125c9 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -891,7 +891,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m try options.env_option_overrides.ensureUnusedCapacity(arena, 16); if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { setFlag(&options, "xla_dump_to", xla_dump_to); - setFlag(&options, "xla_dump_hlo_as_dot", true); + setFlag(&options, "xla_dump_hlo_as_proto", true); if (platform.compilation_options.xla_dump_fusion_visualization) { setFlag(&options, "xla_dump_fusion_visualization", true); } From d2aa8c8398a408b5467197816cb2fff7401b2be2 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 4 Feb 2025 17:17:05 +0100 Subject: [PATCH 24/27] aio.awaitAll --- zml/aio.zig | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/zml/aio.zig b/zml/aio.zig index 07f7c2a9..a367f56e 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -561,7 +561,7 @@ pub fn loadModelBuffersWithPrefix( return res; } -/// deprecated, use zml.buffers.deinitAll() +/// Deinit all buffers in the given struct pub fn unloadBuffers(model: anytype) void { zml.meta.visit((struct { fn cb(_: void, buffer: *zml.Buffer) void { @@ -570,6 +570,20 @@ pub fn unloadBuffers(model: anytype) void { }).cb, {}, model); } +/// Await all buffers in the given struct +pub fn awaitAll(buffers: anytype) !void { + var err: ?anyerror = null; + zml.meta.visit((struct { + fn cb(err_ptr: *?anyerror, buffer: *zml.Buffer) void { + _ = buffer.awaitt() catch |e| { + err_ptr.* = e; + }; + } + }).cb, &err, buffers); + + return err orelse {}; +} + fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, buffer_store: BufferStore, obj: anytype, platform: zml.Platform) !void { const err_msg = "visitStructAndLoadBuffer must be called with a pointer to type. Received "; const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { From c06ba80e7c891d5415bc12459e693f582c9d5e50 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 5 Feb 2025 13:31:03 +0000 Subject: [PATCH 25/27] zml/buffer.zig: added Buffer.fromBytes --- zml/buffer.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zml/buffer.zig b/zml/buffer.zig index 9bb6237b..b5d7f01f 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -138,6 +138,11 @@ pub const Buffer = struct { try std.testing.expectEqual(-15, (try values_d.getValue([2][8]i32))[1][7]); } + /// Creates a buffer from the given shape and bytes. + pub fn fromBytes(platform: Platform, shape_: Shape, s: []const u8) !Buffer { + return from(platform, HostBuffer.fromBytes(shape_, s)); + } + /// Creates a Buffer with a single element. pub fn scalar(platform: Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); From a336507857fdf3cfcd03264cd95b4a50b3029504 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 5 Feb 2025 15:20:53 +0000 Subject: [PATCH 26/27] pjrt+zml: rework profiler imports Also expose TraceContainer as it can be useful. --- pjrt/pjrt.zig | 4 +++- pjrt/profiler.zig | 8 ++++---- zml/pjrtx.zig | 6 ++++-- zml/platform.zig | 4 ++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 651a1a55..8fd1b75e 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -6,7 +6,9 @@ const c = @import("c"); const log = std.log.scoped(.pjrt); -pub const Profiler = @import("profiler.zig").Profiler; +pub const profiler = @import("profiler.zig"); + +const Profiler = profiler.Profiler; test { std.testing.refAllDecls(@This()); diff --git a/pjrt/profiler.zig b/pjrt/profiler.zig index 1b0d1a69..9a937382 100644 --- a/pjrt/profiler.zig +++ b/pjrt/profiler.zig @@ -3,7 +3,7 @@ const c = @import("c"); const tsl_proto = @import("//tsl:profiler_options_proto"); const log = std.log.scoped(.@"pjrt/profiler"); -const TraceContainer = @import("convert/trace_container.zig").TraceContainer; +pub const TraceContainer = @import("convert/trace_container.zig").TraceContainer; /// Pjrt Profiler extension pub const Profiler = struct { @@ -60,14 +60,14 @@ pub const Profiler = struct { } pub fn start(self: *Profiler) void { - self.transition("start", .ready, .started); + //self.transition("start", .ready, .started); if (self.api == null) return; var args: c.PLUGIN_Profiler_Start_Args = .{ .profiler = self.inner }; self.check(self.api.?.start.?(&args)) catch unreachable; } pub fn stop(self: *Profiler) void { - self.transition("stop", .started, .stopped); + //self.transition("stop", .started, .stopped); if (self.api == null) return; var args: c.PLUGIN_Profiler_Stop_Args = .{ .profiler = self.inner }; @@ -75,7 +75,7 @@ pub const Profiler = struct { } pub fn collectData(self: *Profiler, allocator: std.mem.Allocator) !ProfilingData { - self.transition("collect_data", .stopped, .done); + //self.transition("collect_data", .stopped, .done); if (self.api == null) return .{ .external = &.{} }; var args: c.PLUGIN_Profiler_CollectData_Args = .{ diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 44042aa1..fb38c24f 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -14,7 +14,7 @@ const Target = @import("platform.zig").Target; const log = std.log.scoped(.zml); -pub const Profiler = pjrt.Profiler; +pub const profiler = pjrt.profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; pub const BufferType = pjrt.BufferType; @@ -30,6 +30,8 @@ pub const SerializeResult = pjrt.SerializeResult; pub const Executable = pjrt.Executable; pub const ExecuteError = ApiError; +const Profiler = profiler.Profiler; + fn InnerMixin(comptime innerT: type) type { return struct { inline fn inner(self: anytype) if (@typeInfo(@TypeOf(self)).Pointer.is_const) *const innerT else *innerT { @@ -120,7 +122,7 @@ pub const Client = opaque { /// Returns the Profiler for this API. /// Not all platform have a profiling api, for those the profiler object will do nothing. /// Platforms with known profiler extensions: cuda, xpu - pub fn getProfiler(self: *const Client, api: *const Api, options: pjrt.Profiler.Options) pjrt.Profiler { + pub fn getProfiler(self: *const Client, api: *const Api, options: Profiler.Options) Profiler { return self.inner().getProfiler(api, options); } }; diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..76acb159 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -82,8 +82,8 @@ pub const Platform = struct { /// Returns the Profiler for this API. /// Not all platform have a profiling api, for those the profiler object will do nothing. /// Platforms with known profiler extensions: cuda, xpu - pub fn getProfiler(self: Platform, options: ?pjrt.Profiler.Options) pjrt.Profiler { - return self.pjrt_client.getProfiler(self.pjrt_api, options orelse pjrt.Profiler.default_options); + pub fn getProfiler(self: Platform, options: ?pjrt.profiler.Profiler.Options) pjrt.profiler.Profiler { + return self.pjrt_client.getProfiler(self.pjrt_api, options orelse pjrt.profiler.Profiler.default_options); } }; From b88ab792840dbccffe985a2fcd9b96dfc579ba2c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Dalido Date: Wed, 12 Feb 2025 10:17:58 +0000 Subject: [PATCH 27/27] async: fix udp listener --- async/async.zig | 6 +++--- async/asyncio.zig | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/async/async.zig b/async/async.zig index 79026da2..cf16f69e 100644 --- a/async/async.zig +++ b/async/async.zig @@ -411,7 +411,7 @@ pub const Socket = struct { }; pub const UDP = struct { - const Inner = aio.TCP; + const Inner = aio.UDP; pub const Reader = std.io.GenericReader(UDP, stdx.meta.FnSignature(UDP.read, null).ReturnErrorSet.?, UDP.read); pub const WriterContext = struct { @@ -431,11 +431,11 @@ pub const Socket = struct { .inner = aio.UDP.init(AsyncThread.current.executor, try xev.UDP.init(addr)), }; try self.inner.udp.bind(addr); - try self.inner.udp.listen(1024); + // try self.inner.udp.listen(1024); return self; } - pub fn read(self: UDP, buf: []u8) !usize { + pub fn read(self: UDP, buf: []u8) !struct { usize, std.net.Address } { return self.inner.read(.{ .slice = buf }); } diff --git a/async/asyncio.zig b/async/asyncio.zig index ab0fa869..a3d86831 100644 --- a/async/asyncio.zig +++ b/async/asyncio.zig @@ -513,10 +513,17 @@ pub const UDP = struct { const ReadResult = xev.ReadError!usize; pub fn read(self: Self, buf: xev.ReadBuffer) !usize { + const size_addr, _ = try self.read_addr(buf); + + return size_addr; + } + + pub fn read_addr(self: Self, buf: xev.ReadBuffer) !struct { usize, std.net.Address } { const ResultT = ReadResult; const Data = struct { result: ResultT = undefined, frame: ?Frame = null, + addr: std.net.Address, fn callback( userdata: ?*@This(), @@ -531,11 +538,12 @@ pub const UDP = struct { _ = l; _ = c; _ = s; - _ = addr; _ = udp; _ = b; const data = userdata.?; data.result = result; + data.addr = addr; + if (data.frame != null) libcoro.xresume(data.frame.?); return .disarm; } @@ -544,12 +552,12 @@ pub const UDP = struct { const loop = self.exec.loop; var s: xev.UDP.State = undefined; var c: xev.Completion = .{}; - var data: Data = .{ .frame = libcoro.xframe() }; + var data: Data = .{ .frame = libcoro.xframe(), .addr = undefined }; self.udp.read(loop, &c, &s, buf, Data, &data, &Data.callback); try waitForCompletion(self.exec, &c); - - return data.result; + std.debug.print("{d}", .{data.addr.in.sa.addr}); + return .{ try data.result, data.addr }; } const WriteResult = xev.WriteError!usize;