From 7980fb51ff970096938c67beca3f0abc73c9cce0 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 7 Jan 2025 22:55:02 +0100 Subject: [PATCH 01/21] zml: make buffers and execute asynchronous This change makes buffer and execution completely asynchronous. This enable easy pipelining of multiple operations. It doesn't map well to `async` because the PJRT API is already asynchronous, so might as well use it as is. --- examples/llama/main.zig | 2 +- examples/mnist/mnist.zig | 4 +-- pjrt/pjrt.zig | 2 +- zml/buffer.zig | 44 +++++++++++++----------- zml/pjrtx.zig | 74 ++++++++++++++++++++++++++++++++++------ 5 files changed, 92 insertions(+), 34 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6c2a9df0..3215f6d6 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -166,7 +166,7 @@ const params = clap.parseParamsComptime( \\--seed random seed (optional) \\--seq-len sequence length \\--create-options platform creation options JSON, defaults to {} - \\--no-llama3 skip prompt template + \\--no-llama3 skip prompt template \\--sharding default: true: sharding on or off ); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c49300a9..7b643db7 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -101,11 +101,11 @@ pub fn asyncMain() !void { const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); + var input = try zml.Buffer.from2(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); + var result: zml.Buffer = mnist.call(.{try input.awaitt()}); defer result.deinit(); log.info( diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9cfa5fb5..651a1a55 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -732,7 +732,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, diff --git a/zml/buffer.zig b/zml/buffer.zig index a9fba67a..f149ef04 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -56,38 +56,39 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - .{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - - frames.appendAssumeCapacity(frame); - } - - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } res._shards.appendAssumeCapacity(pjrt_buffer); } + return res; } + pub fn awaitt(self: Buffer) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.awaitt(self._api); + } + } + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -108,6 +109,11 @@ pub const Buffer = struct { return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); } + pub fn fromSlice2(platform: Platform, dimz: anytype, s: anytype) !Buffer { + const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); + return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); + } + /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 7408e074..cf68dba0 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -17,7 +17,6 @@ const log = std.log.scoped(.zml); pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; -pub const Buffer = pjrt.Buffer; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; @@ -63,13 +62,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return buffer; + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -130,6 +125,62 @@ pub const Client = opaque { } }; +pub const Buffer = opaque { + pub const inner = InnerMixin(pjrt.Buffer).inner; + + pub fn deinit(self: *Buffer, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { + return try self.inner().getDevice(api); + } + + pub fn delete(self: *Buffer, api: *const Api) void { + self.inner().delete(api); + } + + pub fn isDeleted(self: *const Buffer, api: *const Api) bool { + return self.inner().isDeleted(api); + } + + pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { + return self.inner().isOnCpu(api); + } + + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { + return @ptrCast(try self.inner().toHostBuffer(api, dst)); + } + + pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { + return self.inner().getElementType(api); + } + + pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { + return self.inner().getDimensions(api); + } + + pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { + return try self.inner().getUnpaddedDimensions(api); + } + + pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { + return try self.inner().getOnDeviceSizeInBytes(api); + } + + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { + return @ptrCast(self.inner().copyToDevice(api, device)); + } + + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { + return @ptrCast(self.inner().getReadyEvent(api)); + } + + pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { + return try self.inner().getOpaqueDeviceMemoryDataPointer(api); + } +}; + pub const Event = opaque { pub const inner = InnerMixin(pjrt.Event).inner; @@ -145,7 +196,8 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) !void { + pub const await_ = awaitt; + pub fn awaitt(self: *Event, api: *const Api) !void { defer self.deinit(api); if (self.isReady(api)) { @@ -201,13 +253,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 8a5b73e70bb03d399eeba4f63bfd975967191c74 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 20 Jan 2025 19:12:39 +0100 Subject: [PATCH 02/21] zml/aio: wait for buffer readiness in loadBuffers --- zml/aio.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/aio.zig b/zml/aio.zig index 55155c48..2f20f4f6 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -589,7 +589,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from(platform, buf_with_metadata); + obj.* = try (try zml.Buffer.from(platform, buf_with_metadata)).awaitt(); } else { return error.BufferNotFound; }; From 9ca43972d85a153b0ba2ff952268bac2fa7f1c09 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Mon, 20 Jan 2025 19:47:56 +0100 Subject: [PATCH 03/21] llama: fix async toHost usage when fetching token from device. --- examples/llama/main.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 3215f6d6..2517fa19 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -128,9 +128,11 @@ pub fn generateText( tracer.frameEnd(frame_id, try std.fmt.bufPrintZ(tracer_buffer, "Generated token {}/{}", .{ i + 1, output_tokens_len })); - // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + // extract the generated token from the buffer, async + var fut_tok_buf = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + _ = try fut_tok_buf.awaitt(); const generated_token = generated_token_buffer[0]; + // de-tokenize generated token into a string const chunk = try tokenizer_decoder.next(@intCast(generated_token)) orelse unreachable; num_tokens_generated = i; From 9517504d02e82ad348d3c5471fc2068764a78e4c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 10:19:42 +0000 Subject: [PATCH 04/21] zml/aio: await weights transfers --- zml/aio.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zml/aio.zig b/zml/aio.zig index 2f20f4f6..90206259 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -589,7 +589,8 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try (try zml.Buffer.from(platform, buf_with_metadata)).awaitt(); + obj.* = try zml.Buffer.from(platform, buf_with_metadata); + _ = try obj.awaitt(); } else { return error.BufferNotFound; }; From 2612952a440268bb4945bd141dd23996ff3c5176 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 10:20:09 +0000 Subject: [PATCH 05/21] zml: add Buffer.Shard type --- zml/buffer.zig | 66 +++++++++++++++++++++++++++++++++----------------- zml/exe.zig | 4 +-- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index f149ef04..7d2afa1d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -27,12 +27,29 @@ const log = std.log.scoped(.zml); /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { + pub const Shard = struct { + api: *const pjrt.Api, + buffer: *pjrt.Buffer, + ready_event: ?*pjrt.Event = null, + ready: bool = false, + + pub fn awaitt(self: *Shard) !void { + if (self.ready) { + return; + } + if (self.ready_event orelse self.buffer.getReadyEvent(self.api)) |ev| { + try ev.awaitt(self.api); + } + self.ready = true; + } + }; + _shape: Shape, _api: *const pjrt.Api, _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; - pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const Shards = std.BoundedArray(Shard, MAX_NUM_SHARDS); /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { @@ -71,20 +88,19 @@ pub const Buffer = struct { .device = devices[i], .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - if (event) |ev| { - ev.deinit(platform.pjrt_api); - } - res._shards.appendAssumeCapacity(pjrt_buffer); + res._shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + .ready_event = event, + }); } return res; } - pub fn awaitt(self: Buffer) !Buffer { - for (self._shards.constSlice()) |buffer| { - if (buffer.getReadyEvent(self._api)) |ev| { - try ev.awaitt(self._api); - } + pub fn awaitt(self: *Buffer) !*Buffer { + for (self._shards.slice()) |*shard| { + try shard.awaitt(); } return self; } @@ -94,7 +110,12 @@ pub const Buffer = struct { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); + for (pjrt_buffers) |pjrt_buffer| { + shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + }); + } return .{ ._api = platform.pjrt_api, ._shape = shape_, @@ -251,11 +272,12 @@ pub const Buffer = struct { /// The returned `HostBuffer` doesn't own the memory. pub fn toHost(self: Buffer, output: []u8) !HostBuffer { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, output); + var hb = HostBuffer.fromBytes(self.shape(), output); + hb._event = maybe_event; + hb._api = self._api; + hb._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return hb; } /// Copies the content of the Buffer to the host. @@ -263,10 +285,10 @@ pub const Buffer = struct { pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { const output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); - if (maybe_event) |event| { - try event.await_(self._api); - } + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, @constCast(output.data)); + output._event = maybe_event; + output._api = self._api; + output._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; return output; } @@ -274,8 +296,8 @@ pub const Buffer = struct { /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *const Buffer) void { - for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._api); + for (self._shards.constSlice()) |shard| { + shard.buffer.deinit(self._api); } } diff --git a/zml/exe.zig b/zml/exe.zig index b34d5e9b..5105184a 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -316,7 +316,7 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { const model_sharding = ctx.buffers.len; stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {}-sharded tensor into a {}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); for (buffer._shards.constSlice(), 0..) |shard, d| { - ctx.buffers[d][ctx.index] = shard; + ctx.buffers[d][ctx.index] = shard.buffer; } ctx.index += 1; } @@ -344,7 +344,7 @@ fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Bu ctx.index += 1; if (i >= ctx.buffer_shapes.len) return; - var shards: Buffer.Shards = .{}; + var shards: std.BoundedArray(*pjrt.Buffer, Buffer.MAX_NUM_SHARDS) = .{}; for (ctx.buffers) |buff| { shards.appendAssumeCapacity(buff[i]); } From 85698b515e28ed2233016db38ffab3a4a61b1faf Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 10:20:28 +0000 Subject: [PATCH 06/21] zml/exe: don't await for shard after execution --- zml/exe.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index 5105184a..b053f8bf 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -210,11 +210,11 @@ pub const BaseExe = struct { .non_donatable_input_indices = &.{}, }) catch unreachable; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn serialize(self: BaseExe, writer: anytype) !void { From 9a683a87b28bf076d2448108fe35b897e6b26022 Mon Sep 17 00:00:00 2001 From: Rene Schallner Date: Tue, 21 Jan 2025 12:15:27 +0100 Subject: [PATCH 07/21] zml(buffer): fix const bug in toHostAlloc --- zml/buffer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 7d2afa1d..43e8f7cc 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -283,7 +283,7 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); + var output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, @constCast(output.data)); output._event = maybe_event; From 1141fa7f9d40dab8f7557feaad29ef49abdc7543 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 21 Jan 2025 15:23:34 +0100 Subject: [PATCH 08/21] zml,mnist: fix async buffers handling --- examples/mnist/mnist.zig | 4 ++-- zml/buffer.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 7b643db7..c49300a9 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -101,11 +101,11 @@ pub fn asyncMain() !void { const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from2(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); + var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{try input.awaitt()}); + var result: zml.Buffer = mnist.call(.{input}); defer result.deinit(); log.info( diff --git a/zml/buffer.zig b/zml/buffer.zig index 43e8f7cc..bacfd39e 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -260,7 +260,7 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { try event.await_(self._api); } From b3fdf197fed6f6d09ca6caeb42e53702ce14e8d5 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 20 Jan 2025 14:46:22 +0100 Subject: [PATCH 09/21] zml: add asynchronous HostBuffer event --- zml/buffer.zig | 2 +- zml/hostbuffer.zig | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index bacfd39e..eeac255c 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -262,7 +262,7 @@ pub const Buffer = struct { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.awaitt(self._api); } return res; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index cd7e3a60..93daef2c 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -1,6 +1,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); + const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -22,6 +24,9 @@ pub const HostBuffer = struct { managed: u5, unmanaged, } = .unmanaged, + _ready: bool = true, + _api: ?*const pjrt.Api = null, + _event: ?*pjrt.Event = null, /// Allocates a HostBuffer with the given shape. /// The memory is left undefined. @@ -255,6 +260,20 @@ pub const HostBuffer = struct { }; } + pub fn awaitt(self: *HostBuffer) !*HostBuffer { + if (self._ready) { + return self; + } + if (self._event) |ev| { + if (self._api) |api| { + try ev.awaitt(api); + } else { + stdx.debug.panic("Got an event but no API. This should not happen.", .{}); + } + } + return self; + } + pub fn format( self: HostBuffer, comptime fmt: []const u8, From 39a64d1041ccc0c70caf249319581a896111618a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 26 Jan 2025 12:15:35 +0000 Subject: [PATCH 10/21] zml/buffer: await constant buffer creation --- zml/buffer.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index eeac255c..86fa7f8a 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -195,7 +195,9 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, .data = &bytes }; - return try from(platform, host_buffer); + var buf = try from(platform, host_buffer); + _ = try buf.awaitt(); + return buf; } test constant { From 6e3879546bc1bf848afff9ede21cd927b89ceb0b Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 26 Jan 2025 12:16:04 +0000 Subject: [PATCH 11/21] zml/hostbuffer: is buffer is ready, mark it ready --- zml/hostbuffer.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 93daef2c..da96cff9 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -271,6 +271,7 @@ pub const HostBuffer = struct { stdx.debug.panic("Got an event but no API. This should not happen.", .{}); } } + self._ready = true; return self; } From f4eb7594b420ad798ce3c0c3ab3d2d24ed67c8df Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 26 Jan 2025 12:16:32 +0000 Subject: [PATCH 12/21] zml/module: disable CUDA libnvptx for older plugin --- zml/module.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 0c1bc19f..a003896d 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -905,8 +905,8 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables setFlag(&options, "xla_gpu_enable_triton_gemm", false); setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); - setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); - setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); + // setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); + // setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); From d9f2fa8d192d379d0174cca80890b03399ea97c1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 27 Jan 2025 17:56:31 +0000 Subject: [PATCH 13/21] zml/tensor.zig: add maximum as scatter operation --- zml/tensor.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zml/tensor.zig b/zml/tensor.zig index 23ab003b..f2e37497 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2419,6 +2419,10 @@ pub const Tensor = struct { /// of the operator which is backend specific. update_fn: *const fn (Tensor, Tensor) Tensor = increment, + pub fn maximum(old_value: Tensor, new_value: Tensor) Tensor { + return old_value.maximum(new_value.convert(old_value.dtype())); + } + pub fn increment(old_value: Tensor, new_value: Tensor) Tensor { return old_value.add(new_value.convert(old_value.dtype())); } From 01c54e0e72c8ed495a66deaf758e0fe84745f5e3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 29 Jan 2025 10:04:12 +0100 Subject: [PATCH 14/21] zml: exe should collect events --- zml/buffer.zig | 29 +++++++----------- zml/exe.zig | 80 ++++++++++++++++++++++++------------------------- zml/testing.zig | 2 ++ 3 files changed, 51 insertions(+), 60 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 86fa7f8a..bb343d7d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -28,6 +28,8 @@ const log = std.log.scoped(.zml); /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { pub const Shard = struct { + // TODO: remove api from here + // Also we should use SoA instead of AoS because of the "ready" bool api: *const pjrt.Api, buffer: *pjrt.Buffer, ready_event: ?*pjrt.Event = null, @@ -105,24 +107,6 @@ pub const Buffer = struct { return self; } - /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { - stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); - stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); - var shards: Shards = .{}; - for (pjrt_buffers) |pjrt_buffer| { - shards.appendAssumeCapacity(.{ - .api = platform.pjrt_api, - .buffer = pjrt_buffer, - }); - } - return .{ - ._api = platform.pjrt_api, - ._shape = shape_, - ._shards = shards, - }; - } - /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromSlice(platform: Platform, dimz: anytype, s: anytype) !Buffer { @@ -222,6 +206,7 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. + /// The returned buffer will be considered as ready to read. pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const anyopaque, device_data: *anyopaque) !Buffer { const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { var res: [Shape.MAX_RANK]i64 = undefined; @@ -249,7 +234,13 @@ pub const Buffer = struct { }); var shards: Shards = .{}; - shards.appendAssumeCapacity(pjrt_buffer); + shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + // We don't have event here so we assume it's ready + .ready_event = null, + .ready = true, + }); return .{ ._api = platform.pjrt_api, ._shape = shape_, diff --git a/zml/exe.zig b/zml/exe.zig index b053f8bf..6a5a226b 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -140,6 +140,8 @@ pub const BaseExe = struct { /// Pre-allocated slice of buffers to use as outputs when the module is called. output_per_device: []const [*]*pjrt.Buffer, + events_per_device: []?*pjrt.Event, + /// Number of buffers already fed to the executable. ready_buffer_count: u32, @@ -161,8 +163,8 @@ pub const BaseExe = struct { const n_out = args.result_shapes.len; const n_devices = args.n_devices; // Allocate once for all the *pjrt.Buffer we need to store ... - const all_buffers = try allocator.alloc(*pjrt.Buffer, (args.n_in + n_out) * n_devices); - const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ args.n_in * n_devices, n_out * n_devices }); + const all_buffers = try allocator.alloc(*pjrt.Buffer, (args.n_in + n_out + 1) * n_devices); + const all_input_buffers, const all_output_buffers, const events_per_device = splitBuffer(*pjrt.Buffer, all_buffers, .{ args.n_in * n_devices, n_out * n_devices, n_devices }); // ... and once for all the [*]*pjrt.Buffer. const all_per_device = try allocator.alloc([*]*pjrt.Buffer, 2 * n_devices); @@ -181,6 +183,7 @@ pub const BaseExe = struct { .num_devices = args.n_devices, .input_per_device = input_per_device, .output_per_device = output_per_device, + .events_per_device = @ptrCast(events_per_device), .result_shapes = try allocator.dupe(Shape, args.result_shapes), ._arena = arena, }; @@ -196,25 +199,16 @@ pub const BaseExe = struct { } pub fn _unsafeCall(self: BaseExe) void { - var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; - const sharding = self.platform.sharding(); - self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.input_buffer_count, .results = self.output_per_device, - .events = events[0..sharding.num_partitions], + .events = self.events_per_device, // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, }) catch unreachable; - - // for (events[0..sharding.num_partitions]) |e| { - // if (e) |ev| { - // ev.await_(self.platform.pjrt_api) catch unreachable; - // } - // } } pub fn serialize(self: BaseExe, writer: anytype) !void { @@ -235,13 +229,28 @@ pub const BaseExe = struct { self.ready_buffer_count += n; } - pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { - var shards: Buffer.Shards = .{}; - for (self.output_per_device) |dev_out| { - shards.appendAssumeCapacity(dev_out[i]); - } + pub fn numOutputs(self: BaseExe) u32 { + return @intCast(self.result_shapes.len); + } - return Buffer.fromPjrtBuffers(self.platform, self.result_shapes[i], shards.constSlice()); + /// Takes pjrt.Buffer from each device and make it into a Buffer. + /// Set the event of the buffer to the event of the call. + pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { + std.debug.assert(i < self.numOutputs()); + var shards: Buffer.Shards = .{}; + for (0..self.num_devices) |dev| { + shards.appendAssumeCapacity(.{ + .api = self.platform.pjrt_api, + .buffer = self.output_per_device[dev][i], + .ready_event = self.events_per_device[dev], + .ready = false, + }); + } + return .{ + ._shape = self.result_shapes[i], + ._api = self.platform.pjrt_api, + ._shards = shards, + }; } }; @@ -283,7 +292,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { std.debug.assert(total_ready == self.inner.input_buffer_count); self.inner._unsafeCall(); var result: Bufferized(ReturnT) = undefined; - assignRawBuffers(&result, self.inner.platform, self.inner.output_per_device, self.inner.result_shapes); + assignRawBuffers(&result, self.inner); return result; } }; @@ -325,31 +334,20 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { } /// Visit the given struct and override tensors by creating a new one using the provided PJRT buffers. -fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Buffer, buffer_shapes: []Shape) void { +fn assignRawBuffers(v: anytype, exe: BaseExe) void { const LocalContext = struct { - index: u32, - platform: Platform, - buffers: []const [*]*pjrt.Buffer, - buffer_shapes: []Shape, - }; - var local_ctx: LocalContext = .{ - .index = 0, - .platform = platform, - .buffers = buffers, - .buffer_shapes = buffer_shapes, - }; - meta.visit((struct { - fn cb(ctx: *LocalContext, buffer: *Buffer) void { + index: u32 = 0, + exe: *const BaseExe, + + fn cb(ctx: *@This(), buffer: *Buffer) void { const i = ctx.index; ctx.index += 1; - if (i >= ctx.buffer_shapes.len) return; + if (i >= ctx.exe.result_shapes.len) return; - var shards: std.BoundedArray(*pjrt.Buffer, Buffer.MAX_NUM_SHARDS) = .{}; - for (ctx.buffers) |buff| { - shards.appendAssumeCapacity(buff[i]); - } - buffer.* = Buffer.fromPjrtBuffers(ctx.platform, ctx.buffer_shapes[i], shards.constSlice()); + buffer.* = ctx.exe.getOutputBuffer(i); } - }).cb, &local_ctx, v); - stdx.debug.internalAssert(local_ctx.index == buffer_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ buffers.len, @typeName(@TypeOf(v)), local_ctx.index }); + }; + var local_ctx: LocalContext = .{ .exe = &exe }; + meta.visit(LocalContext.cb, &local_ctx, v); + stdx.debug.internalAssert(local_ctx.index == exe.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ exe.result_shapes.len, @typeName(@TypeOf(v)), local_ctx.index }); } diff --git a/zml/testing.zig b/zml/testing.zig index e686b603..4ae4140c 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -41,11 +41,13 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { .{ try left_.toHostAlloc(allocator), true } else .{ left_, false }; + _ = try left.awaitt(); var right: zml.HostBuffer, const should_free_right = if (@TypeOf(right_) == zml.Buffer) .{ try right_.toHostAlloc(allocator), true } else .{ right_, false }; + _ = try right.awaitt(); defer { if (should_free_left) left.deinit(allocator); From 6050d4f032a144cde20b9f3322d152c08564c56c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 29 Jan 2025 10:27:16 +0100 Subject: [PATCH 15/21] zml: remove api from Shard --- zml/buffer.zig | 15 ++++++--------- zml/exe.zig | 1 - 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index bb343d7d..71052895 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -28,19 +28,18 @@ const log = std.log.scoped(.zml); /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { pub const Shard = struct { - // TODO: remove api from here - // Also we should use SoA instead of AoS because of the "ready" bool - api: *const pjrt.Api, + // TODO: we should use SoA instead of AoS because of the "ready" bool. + // the Buffer struct is already huge. buffer: *pjrt.Buffer, ready_event: ?*pjrt.Event = null, ready: bool = false, - pub fn awaitt(self: *Shard) !void { + pub fn awaitt(self: *Shard, api: *const pjrt.Api) !void { if (self.ready) { return; } - if (self.ready_event orelse self.buffer.getReadyEvent(self.api)) |ev| { - try ev.awaitt(self.api); + if (self.ready_event orelse self.buffer.getReadyEvent(api)) |ev| { + try ev.awaitt(api); } self.ready = true; } @@ -91,7 +90,6 @@ pub const Buffer = struct { .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); res._shards.appendAssumeCapacity(.{ - .api = platform.pjrt_api, .buffer = pjrt_buffer, .ready_event = event, }); @@ -102,7 +100,7 @@ pub const Buffer = struct { pub fn awaitt(self: *Buffer) !*Buffer { for (self._shards.slice()) |*shard| { - try shard.awaitt(); + try shard.awaitt(self._api); } return self; } @@ -235,7 +233,6 @@ pub const Buffer = struct { var shards: Shards = .{}; shards.appendAssumeCapacity(.{ - .api = platform.pjrt_api, .buffer = pjrt_buffer, // We don't have event here so we assume it's ready .ready_event = null, diff --git a/zml/exe.zig b/zml/exe.zig index 6a5a226b..3c64f7f9 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -240,7 +240,6 @@ pub const BaseExe = struct { var shards: Buffer.Shards = .{}; for (0..self.num_devices) |dev| { shards.appendAssumeCapacity(.{ - .api = self.platform.pjrt_api, .buffer = self.output_per_device[dev][i], .ready_event = self.events_per_device[dev], .ready = false, From bc514c191d59786dc572dea276e9543ba58d8b55 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 30 Jan 2025 18:00:13 +0100 Subject: [PATCH 16/21] revert zml: exe should collect events --- zml/exe.zig | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index 3c64f7f9..db87ddf7 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -140,8 +140,6 @@ pub const BaseExe = struct { /// Pre-allocated slice of buffers to use as outputs when the module is called. output_per_device: []const [*]*pjrt.Buffer, - events_per_device: []?*pjrt.Event, - /// Number of buffers already fed to the executable. ready_buffer_count: u32, @@ -163,8 +161,8 @@ pub const BaseExe = struct { const n_out = args.result_shapes.len; const n_devices = args.n_devices; // Allocate once for all the *pjrt.Buffer we need to store ... - const all_buffers = try allocator.alloc(*pjrt.Buffer, (args.n_in + n_out + 1) * n_devices); - const all_input_buffers, const all_output_buffers, const events_per_device = splitBuffer(*pjrt.Buffer, all_buffers, .{ args.n_in * n_devices, n_out * n_devices, n_devices }); + const all_buffers = try allocator.alloc(*pjrt.Buffer, (args.n_in + n_out) * n_devices); + const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ args.n_in * n_devices, n_out * n_devices }); // ... and once for all the [*]*pjrt.Buffer. const all_per_device = try allocator.alloc([*]*pjrt.Buffer, 2 * n_devices); @@ -183,7 +181,6 @@ pub const BaseExe = struct { .num_devices = args.n_devices, .input_per_device = input_per_device, .output_per_device = output_per_device, - .events_per_device = @ptrCast(events_per_device), .result_shapes = try allocator.dupe(Shape, args.result_shapes), ._arena = arena, }; @@ -199,11 +196,15 @@ pub const BaseExe = struct { } pub fn _unsafeCall(self: BaseExe) void { + var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; + const sharding = self.platform.sharding(); + self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.input_buffer_count, .results = self.output_per_device, - .events = self.events_per_device, + // Create a local inste + .events = events[0..sharding.num_partitions], // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? @@ -241,7 +242,7 @@ pub const BaseExe = struct { for (0..self.num_devices) |dev| { shards.appendAssumeCapacity(.{ .buffer = self.output_per_device[dev][i], - .ready_event = self.events_per_device[dev], + .ready_event = null, .ready = false, }); } From 459211563896c1103754b6b4b9ba7f6ed54d9eb3 Mon Sep 17 00:00:00 2001 From: Rene Schallner <30892199+renerocksai@users.noreply.github.com> Date: Mon, 27 Jan 2025 17:06:03 +0100 Subject: [PATCH 17/21] llama: fix dims & dtypes -> fix #170 (#171) This PR fixes issue #170 , making Llama run again on CUDA and RoCM. - Latest Llama changes had introduced an unnecessary batching dimension: fixed - Ops returning indices return i32 yet we now use u32 as token indices -> dtype conversion requred: fixed Tested on CUDA. --- examples/llama/llama.zig | 10 ++++------ examples/llama/main.zig | 16 ++++++++-------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 0f57d877..6f6594f2 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -86,7 +86,6 @@ pub const LlamaLM = struct { rng: Tensor.Rng, ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); - var tokens = tokens_.withPartialTags(.{.s}); const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); tokens, const new_rng = self.sampleTokens(self.lm_head, tokens, out, rng, self.gen_opts); @@ -115,7 +114,7 @@ pub const LlamaLM = struct { logits = logits.rename(.{ .d = .voc }); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); - return .{ next_tokens.reuseBuffer(tokens_), new_rng }; + return .{ next_tokens.convert(tokens_.dtype()).reuseBuffer(tokens_), new_rng }; } pub fn increment(_: u8, token_index: Tensor) Tensor { @@ -163,7 +162,6 @@ pub const Llama = struct { /// Returns result and updated KV cache. pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { const embeds = embed(self.embed_tokens, tokens); - var hidden = embeds; var updated_kv_cache = kv_cache; @@ -280,12 +278,12 @@ pub const SelfAttn = struct { // Note: in Pytorch it would be very inefficient to generate the full attn_mask, // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = x.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .b = token_index.shape().dim(0), .coord = 1 }), .{}); + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = x.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); // In self-attention, .s axis is used both for keys and queries. const pos_index = b: { - const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .b = token_index.shape().dim(0), .s = x.dim(.s) }, token_index.dtype())); - break :b temp.add(token_index.withTags(.{.b}).broad(temp.shape())); + const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); }; q = zml.nn.rope(q, pos_index, self.rope_opts); diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 2517fa19..5520498b 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -85,7 +85,8 @@ pub fn generateText( // prepare device buffers for the prefill tokens and the index var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); defer prefill_tokens.deinit(); - var prefill_token_index = try zml.Buffer.fromSlice(platform, .{}, &[_]u32{0}); + var prefill_token_index = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0); + defer prefill_token_index.deinit(); // init RNG and prefill @@ -97,7 +98,7 @@ pub fn generateText( // Prepare for token-by-token generation var first_token_hostbuffer = [_]u32{prompt_tok[prompt_tok.len - 1]}; // start with the prompt's last token - var current_token = try zml.Buffer.fromSlice(platform, .{}, &first_token_hostbuffer); + var current_token = try zml.Buffer.fromSlice(platform, .{1}, &first_token_hostbuffer); defer current_token.deinit(); // Here we will copy the generated token from device @@ -121,6 +122,7 @@ pub fn generateText( // current token index needs to go into a zml.Buffer const token_index_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; const token_index = try zml.Buffer.fromSlice(platform, .{}, token_index_buffer); + defer token_index.deinit(); // call to generate the next token @@ -258,13 +260,11 @@ pub fn asyncMain() !void { const dims = model_instance.model.shape(); const dtype = model_instance.model.embed_tokens.weight.dtype(); - const batch_size = 1; + const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); + const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); + const token_idx_shape = zml.Shape.init(.{}, .u32); - const tokens_shape_prefill = zml.Shape.init(.{ .b = batch_size, .s = llama_options.max_seq_len }, .u32); - const tokens_shape = zml.Shape.init(.{ .b = batch_size, .s = 1 }, .u32); - const token_idx_shape = zml.Shape.init(.{ .b = batch_size }, .u32); - - const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .b = batch_size, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype).withSharding(.{.h}); + const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype).withSharding(.{.h}); const kv_cache_shape: zml.ShapeOf(llama.KvCache) = llama.KvCache.initShape(kv_shape); const rng_shape = zml.Tensor.Rng.shape(); From be97bf79eae7b123aa614ba2d1c450683a526524 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 30 Jan 2025 14:32:40 +0000 Subject: [PATCH 18/21] wip --- main.py | 39 ++++++++ pjrt/pjrt.zig | 122 ++++++++++++++++++++++++- runtimes/cuda/BUILD.bazel | 5 + runtimes/cuda/cuda.zig | 3 + runtimes/cuda/libpjrt_cuda.BUILD.bazel | 3 +- zml/buffer.zig | 119 ++++++++++++++++++++++-- zml/exe.zig | 3 + zml/pjrtx.zig | 38 +++++++- zml/platform.zig | 3 + 9 files changed, 325 insertions(+), 10 deletions(-) create mode 100644 main.py diff --git a/main.py b/main.py new file mode 100644 index 00000000..f3bf55bd --- /dev/null +++ b/main.py @@ -0,0 +1,39 @@ +import jax +import jax.numpy as jnp +from ctypes import cdll +libcudart = cdll.LoadLibrary('libcudart.so') + +jax.config.update("jax_traceback_in_locations_limit", -1) + +@jax.jit +def benchmark(a, b): + return jnp.dot(a, b) + +# Main function +def main(): + size = 8192 + dtype = jnp.bfloat16 + + rng_key = jax.random.PRNGKey(0) + a = jax.random.uniform(rng_key, shape=(size, size), dtype=dtype) + b = jax.random.uniform(rng_key, shape=(size, size), dtype=dtype) + + print("Compiling the benchmark function...") + compiled_benchmark = jax.jit(benchmark) + + print("Running warm-up execution...") + compiled_benchmark(a, b) + + print("Running benchmark...") + + # with jax.profiler.trace("/home/zml/zml-hugo/jax-trace", create_perfetto_link=True): + libcudart.cudaProfilerStart() + result = compiled_benchmark(a, b) + result2 = compiled_benchmark(result, b) + jax.device_get(result) + jax.device_get(result2) + libcudart.cudaProfilerStop() + +if __name__ == "__main__": + main() + diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 651a1a55..cab2533a 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -324,6 +324,7 @@ pub const Client = opaque { byte_strides: ?[]const i64, device: *const Device, host_buffer_semantics: HostBufferSemantics, + memory: *const Memory, }; pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { @@ -337,7 +338,7 @@ pub const Client = opaque { .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), .device = @ptrCast(@constCast(args.device)), - .memory = null, // TODO + // .memory = @ptrCast(@constCast(args.memory)), .device_layout = null, // TODO .done_with_host_buffer = null, .buffer = null, @@ -398,6 +399,16 @@ pub const Client = opaque { }); return @ptrCast(ret.buffer.?); } + + pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { + const ret = api.call(.PJRT_Client_AddressableMemories, .{ + .client = self.inner(), + }) catch unreachable; + if (ret.addressable_memories) |memories| { + return @constCast(@ptrCast(memories[0..ret.num_addressable_memories])); + } + return &.{}; + } }; pub const Device = opaque { @@ -423,6 +434,16 @@ pub const Device = opaque { }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) []*const Memory { + const ret = api.call(.PJRT_Device_AddressableMemories, .{ + .device = self.inner(), + }) catch unreachable; + if (ret.memories) |memories| { + return @ptrCast(memories[0..ret.num_memories]); + } + return &.{}; + } }; pub const DeviceDescription = opaque { @@ -701,6 +722,31 @@ pub const Buffer = opaque { return @ptrCast(ret.event); } + pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64, size: i64) ApiError!?*Event { + const ret = try api.call(.PJRT_Buffer_CopyRawToHost, .{ + .buffer = self.inner(), + .dst = @ptrCast(dst.ptr), + .offset = offset, + .transfer_size = size, + }); + return @ptrCast(ret.event); + } + + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!?*Buffer { + const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ + .buffer = self.inner(), + .dst_memory = @ptrCast(@constCast(dst_memory)), + }); + return @ptrCast(ret.dst_buffer); + } + + pub fn getMemory(self: *const Buffer, api: *const Api) *const Memory { + const ret = api.call(.PJRT_Buffer_Memory, .{ + .buffer = self.inner(), + }) catch unreachable; + return @ptrCast(ret.memory); + } + pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { const ret = api.call(.PJRT_Buffer_ElementType, .{ .buffer = self.inner(), @@ -753,6 +799,18 @@ pub const Buffer = opaque { }); return ret.device_memory_ptr.?; } + + pub fn increaseExternalReferenceCount(self: *const Buffer, api: *const Api) ApiError!void { + _ = try api.call(.PJRT_Buffer_IncreaseExternalReferenceCount, .{ + .buffer = self.inner(), + }); + } + + pub fn decreaseExternalReferenceCount(self: *const Buffer, api: *const Api) ApiError!void { + _ = try api.call(.PJRT_Buffer_DecreaseExternalReferenceCount, .{ + .buffer = self.inner(), + }); + } }; pub const Event = opaque { @@ -793,6 +851,68 @@ pub const Event = opaque { } }; +pub const Memory = opaque { + pub const Kind = enum { + device, + pinned_host, + unpinned_host, + }; + + const inner = InnerMixin(c.PJRT_Memory).inner; + + pub fn id(self: *const Memory, api: *const Api) usize { + const ret = api.call(.PJRT_Memory_Id, .{ + .memory = self.inner(), + }) catch unreachable; + return @intCast(ret.id); + } + + pub fn kind(self: *const Memory, api: *const Api) Kind { + const ret = api.call(.PJRT_Memory_Kind, .{ + .memory = self.inner(), + }) catch unreachable; + const kind_ = ret.kind orelse unreachable; + return std.meta.stringToEnum(Kind, kind_[0..ret.kind_size]) orelse unreachable; + } + + pub fn kindId(self: *const Memory, api: *const Api) usize { + const ret = api.call(.PJRT_Memory_Kind_Id, .{ + .memory = self.inner(), + }) catch unreachable; + return @bitCast(ret.kind_id); + } + + pub fn debugString(self: *const Memory, api: *const Api) []const u8 { + const ret = api.call(.PJRT_Memory_DebugString, .{ + .memory = self.inner(), + }) catch unreachable; + if (ret.debug_string) |debug_string| { + return debug_string[0..ret.debug_string_size]; + } + return &.{}; + } + + pub fn toString(self: *const Memory, api: *const Api) []const u8 { + const ret = api.call(.PJRT_Memory_ToString, .{ + .memory = self.inner(), + }) catch unreachable; + if (ret.to_string) |to_string| { + return to_string[0..ret.to_string_size]; + } + return &.{}; + } + + pub fn addressableByDevices(self: *const Memory, api: *const Api) []*Device { + const ret = api.call(.PJRT_Memory_AddressableByDevices, .{ + .event = self.inner(), + }) catch unreachable; + if (ret.devices) |devices| { + return devices[0..ret.num_devices]; + } + return &.{}; + } +}; + pub const NamedValue = extern struct { comptime { std.debug.assert(@sizeOf(NamedValue) == @sizeOf(c.PJRT_NamedValue)); diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index a7f8bf94..6b50f575 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -1,5 +1,10 @@ load("@rules_zig//zig:defs.bzl", "zig_library") +exports_files( + ["libpjrt_cuda.so"], + visibility = ["//visibility:public"], +) + cc_library( name = "zmlxcuda_lib", srcs = ["zmlxcuda.c"], diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index bc749693..de9dac74 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -62,6 +62,9 @@ pub fn load() !*const pjrt.Api { log.warn("Detected {s} in LD_LIBRARY_PATH. This can lead to undefined behaviors and crashes", .{nvidiaLibsPath}); } + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB", "2", 1); + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_PREALLOCATE", "true", 1); + // CUDA path has to be set _before_ loading the PJRT plugin. // See https://github.com/openxla/xla/issues/21428 try setupXlaGpuCudaDirFlag(); diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index e595ffdf..62f24dd1 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -25,8 +25,7 @@ copy_to_directory( cc_import( name = "libpjrt_cuda", data = [":sandbox"], - shared_library = "jax_plugins/xla_cuda12/xla_cuda_plugin.so", - soname = "libpjrt_cuda.so", + shared_library = "@zml//runtimes/cuda:libpjrt_cuda.so", add_needed = ["libzmlxcuda.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", diff --git a/zml/buffer.zig b/zml/buffer.zig index 71052895..346f3674 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -42,11 +42,13 @@ pub const Buffer = struct { try ev.awaitt(api); } self.ready = true; + // try self.buffer.increaseExternalReferenceCount(api); } }; _shape: Shape, _api: *const pjrt.Api, + _memory: *const pjrt.Memory, _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; @@ -54,12 +56,6 @@ pub const Buffer = struct { /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { - var res: Buffer = .{ - ._api = platform.pjrt_api, - ._shape = host_buffer.shape(), - ._shards = .{}, - }; - // We shard only on the first axis so that the chunks are still contiguous. // TODO: support more advanced sharding specs stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); @@ -75,6 +71,21 @@ pub const Buffer = struct { const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); const devices = platform.getDevices(); + const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == pjrt.Memory.Kind.device) { + selected_mem = mem; + } + } + + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._shape = host_buffer.shape(), + ._shards = .{}, + ._memory = selected_mem, + }; + for (0..n_partitions) |i| { const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; @@ -88,6 +99,7 @@ pub const Buffer = struct { .byte_strides = byte_strides, .device = devices[i], .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .memory = res._memory, }); res._shards.appendAssumeCapacity(.{ .buffer = pjrt_buffer, @@ -98,6 +110,57 @@ pub const Buffer = struct { return res; } + pub fn from2(platform: Platform, buffer_: []const u8, shape_: Shape) !Buffer { + // We shard only on the first axis so that the chunks are still contiguous. + // TODO: support more advanced sharding specs + stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); + const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + _ = sharding_ax; // autofix + const n_partitions = platform.sharding().num_partitions; + const chunk_size = 0; + _ = chunk_size; // autofix + + const buffer_type = bufferTypeFromDtype(shape_.dtype()); + const byte_strides = shape_.computeStrides().constSlice(); + + const devices = platform.getDevices(); + const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == pjrt.Memory.Kind.device) { + selected_mem = mem; + } + } + + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._shape = shape_, + ._shards = .{}, + ._memory = selected_mem, + }; + + for (0..n_partitions) |i| { + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buffer_, + .buffer_type = buffer_type, + .dims = shape_.dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .memory = res._memory, + }); + + // try pjrt_buffer.increaseExternalReferenceCount(platform.pjrt_api); + + res._shards.appendAssumeCapacity(.{ + .buffer = pjrt_buffer, + .ready_event = event, + }); + } + + return res; + } + pub fn awaitt(self: *Buffer) !*Buffer { for (self._shards.slice()) |*shard| { try shard.awaitt(self._api); @@ -270,6 +333,50 @@ pub const Buffer = struct { return hb; } + const ToHostExOpts = struct { + offset: i64 = 0, + // If not set, the whole buffer starting from offset (default: 0) on device is copied. + size: ?i64 = null, + }; + pub fn toHostEx(self: Buffer, output: []u8, opts: ToHostExOpts) !HostBuffer { + var host_buffer = HostBuffer.fromBytes(self.shape(), output); + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + const size: i64 = opts.size orelse @as(i64, @intCast(output.len)) - opts.offset; + const maybe_event = try self._shards.get(0).buffer.copyRawToHost(self._api, @constCast(host_buffer.data), opts.offset, size); + host_buffer._event = maybe_event; + host_buffer._api = self._api; + host_buffer._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return host_buffer; + } + + pub const MemoryKind = pjrt.Memory.Kind; + pub fn copyToMemory(self: Buffer, platform: Platform, kind: pjrt.Memory.Kind) !Buffer { + const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == kind) { + selected_mem = mem; + } + } + + var shards: Shards = .{}; + for (self._shards.constSlice()) |shard| { + const pjrt_buffer = try shard.buffer.copyToMemory(self._api, selected_mem); + const event = if (pjrt_buffer) |buf| buf.getReadyEvent(self._api) else null; + shards.appendAssumeCapacity(.{ + .buffer = pjrt_buffer.?, + .ready_event = event, + .ready = if (event) |ev| ev.isReady(self._api) else true, + }); + } + return .{ + ._api = self._api, + ._shape = self._shape, + ._shards = shards, + ._memory = selected_mem, + }; + } + /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { diff --git a/zml/exe.zig b/zml/exe.zig index db87ddf7..ba63f238 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -246,10 +246,13 @@ pub const BaseExe = struct { .ready = false, }); } + const output_mem = self.output_per_device[0][0].getMemory(self.platform.pjrt_api); + return .{ ._shape = self.result_shapes[i], ._api = self.platform.pjrt_api, ._shards = shards, + ._memory = output_mem, // todo }; } }; diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index cf68dba0..1fd6f665 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -20,6 +20,7 @@ pub const ErrorCode = pjrt.ErrorCode; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; +pub const Memory = pjrt.Memory; pub const Api = pjrt.Api; pub const NamedValue = pjrt.NamedValue; pub const ClientInitError = pjrt.ClientInitError; @@ -123,12 +124,17 @@ pub const Client = opaque { pub fn getProfiler(self: *const Client, api: *const Api, options: pjrt.Profiler.Options) pjrt.Profiler { return self.inner().getProfiler(api, options); } + + pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { + return self.inner().addressableMemories(api); + } }; pub const Buffer = opaque { pub const inner = InnerMixin(pjrt.Buffer).inner; pub fn deinit(self: *Buffer, api: *const Api) void { + // self.inner().decreaseExternalReferenceCount(api); self.inner().deinit(api); } @@ -152,6 +158,18 @@ pub const Buffer = opaque { return @ptrCast(try self.inner().toHostBuffer(api, dst)); } + pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64, size: i64) ApiError!?*Event { + return @ptrCast(try self.inner().copyRawToHost(api, dst, offset, size)); + } + + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!?*Buffer { + return @ptrCast(try self.inner().copyToMemory(api, dst_memory)); + } + + pub fn getMemory(self: *const Buffer, api: *const Api) *const Memory { + return self.inner().getMemory(api); + } + pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { return self.inner().getElementType(api); } @@ -179,6 +197,14 @@ pub const Buffer = opaque { pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { return try self.inner().getOpaqueDeviceMemoryDataPointer(api); } + + pub fn increaseExternalReferenceCount(self: *Buffer, api: *const Api) ApiError!void { + try self.inner().increaseExternalReferenceCount(api); + } + + pub fn decreaseExternalReferenceCount(self: *Buffer, api: *const Api) ApiError!void { + try self.inner().decreaseExternalReferenceCount(api); + } }; pub const Event = opaque { @@ -201,24 +227,34 @@ pub const Event = opaque { defer self.deinit(api); if (self.isReady(api)) { + std.log.warn("Call isReady already: {any}", .{self}); return; } + var timer = try stdx.time.Timer.start(); + var ctx = struct { err: ?*pjrt.Error = null, event: asynk.threading.ResetEventSingle = .{}, - }{}; + timer: *stdx.time.Timer, + }{ + .timer = &timer, + }; + std.log.warn("Call onReady: {any}", .{self}); try self.inner().onReady(api, &(struct { fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.C) void { const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); + var timer_ = ctx_.timer; ctx_.err = err; ctx_.event.set(); + std.log.warn("Callback called: {}", .{timer_.read()}); } }.call), &ctx); ctx.event.wait(); if (ctx.err) |e| { + log.err("Error waiting for event: {}", .{e}); defer e.deinit(api); return e.getCode(api).toApiError(); } diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..67386303 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -108,6 +108,7 @@ const _CreateOptions = struct { // num_nodes // enable_mock_nccl // mock_gpu_topology + should_stage_host_to_device_transfers: bool = false, pub const Allocator = union(enum) { /// "Best-Fit with Coalescing" algorithm @@ -125,6 +126,8 @@ const _CreateOptions = struct { }; pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.fromBool("should_stage_host_to_device_transfers", self.should_stage_host_to_device_transfers)); + switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); From 743b6ac74b450ef99bfd48d364399eb8aba395c2 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 31 Jan 2025 12:29:28 +0000 Subject: [PATCH 19/21] wip --- examples/llama/main.zig | 32 +++++++++++++-- pjrt/pjrt.zig | 19 +++++++++ runtimes/cuda/cuda.zig | 2 +- zml/aio.zig | 18 +++++--- zml/buffer.zig | 9 +++- zml/pjrtx.zig | 21 ++++++++++ zml/platform.zig | 91 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 181 insertions(+), 11 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 5520498b..2425829b 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -22,6 +22,24 @@ pub const std_options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + pub fn tokenizePromptLlama3(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8) ![]u32 { var tokens = std.ArrayList(u32).init(allocator); var encoder = try tokenizer.encoder(); @@ -131,7 +149,7 @@ pub fn generateText( tracer.frameEnd(frame_id, try std.fmt.bufPrintZ(tracer_buffer, "Generated token {}/{}", .{ i + 1, output_tokens_len })); // extract the generated token from the buffer, async - var fut_tok_buf = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + var fut_tok_buf = try current_token.toHostEx(std.mem.sliceAsBytes(&generated_token_buffer), .{}); _ = try fut_tok_buf.awaitt(); const generated_token = generated_token_buffer[0]; @@ -295,7 +313,10 @@ pub fn asyncMain() !void { }); log.info("\tLoading Llama weights from {?s}...", .{res.args.weights}); + var runtime = Runtime.init() catch unreachable; + _ = runtime.cudaProfilerStart(); var llama_weights = try zml.aio.loadBuffers(llama.LlamaLM, .{ config, llama_options }, ts, model_arena.allocator(), platform); + _ = runtime.cudaProfilerStop(); defer zml.aio.unloadBuffers(&llama_weights); log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); @@ -327,7 +348,10 @@ pub fn asyncMain() !void { const seed = res.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); const skip_llama3_encoding = res.args.@"no-llama3" orelse false; - const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); - // generated text will be printed token by token. - defer allocator.free(generated_text); + for (0..2) |_| { + log.warn("----------------------------------------------", .{}); + const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); + // generated text will be printed token by token. + defer allocator.free(generated_text); + } } diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index cab2533a..bf8f99e3 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -444,6 +444,15 @@ pub const Device = opaque { } return &.{}; } + + pub fn getMemoryByKind(self: *const Device, api: *const Api, kind: Memory.Kind) ?*const Memory { + const memories = self.addressableMemories(api); + return for (memories) |m| blk: { + if (m.kind(api) == kind) { + break :blk m; + } + }; + } }; pub const DeviceDescription = opaque { @@ -813,6 +822,16 @@ pub const Buffer = opaque { } }; +pub const AsyncHostToDeviceTransferManager = opaque { + const inner = InnerMixin(c.PJRT_AsyncHostToDeviceTransferManager).inner; + + pub fn deinit(self: *Event, api: *const Api) void { + _ = api.call(.PJRT_AsyncHostToDeviceTransferManager_Destroy, .{ + .event = self.inner(), + }) catch unreachable; + } +}; + pub const Event = opaque { const inner = InnerMixin(c.PJRT_Event).inner; diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index de9dac74..38a8aa6d 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -62,7 +62,7 @@ pub fn load() !*const pjrt.Api { log.warn("Detected {s} in LD_LIBRARY_PATH. This can lead to undefined behaviors and crashes", .{nvidiaLibsPath}); } - _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB", "2", 1); + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB", "48", 1); _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_PREALLOCATE", "true", 1); // CUDA path has to be set _before_ loading the PJRT plugin. diff --git a/zml/aio.zig b/zml/aio.zig index 90206259..164d3c81 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -547,11 +547,19 @@ pub fn loadModelBuffersWithPrefix( // We copy the shape, and let visitStructAndLoadBuffer write the other fields. // to write them just afterward. var res: zml.Bufferized(Model) = undefined; - try zml.meta.mapAlloc(struct { - pub fn initBuffer(_: void, tensor: zml.Tensor) zml.Buffer { - return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined }; + const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device + var selected_mem: *const zml.pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == zml.pjrt.Memory.Kind.device) { + selected_mem = mem; } - }.initBuffer, allocator, {}, model, &res); + } + + try zml.meta.mapAlloc(struct { + pub fn initBuffer(mem: *const zml.pjrt.Memory, tensor: zml.Tensor) zml.Buffer { + return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined, ._memory = mem }; + } + }.initBuffer, allocator, selected_mem, model, &res); var prefix_builder: PrefixBuilder = .{}; try prefix_builder.push(allocator, prefix); @@ -589,7 +597,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from(platform, buf_with_metadata); + obj.* = try zml.Buffer.from2(platform, host_buffer.data, obj._shape); _ = try obj.awaitt(); } else { return error.BufferNotFound; diff --git a/zml/buffer.zig b/zml/buffer.zig index 346f3674..abe33241 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -42,7 +42,6 @@ pub const Buffer = struct { try ev.awaitt(api); } self.ready = true; - // try self.buffer.increaseExternalReferenceCount(api); } }; @@ -161,6 +160,14 @@ pub const Buffer = struct { return res; } + pub fn dataInMemory(self: Buffer) ![]const u8 { + const shard_buffer = self._shards.get(0).buffer; + const memoryDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); + _ = memoryDataPointer; // autofix + stdx.debug.assert(self._shards.len == 1, "TODO: support sharded Buffer -> Host transfer", .{}); + return try self._shards.get(0).buffer.dataInMemory(self._api); + } + pub fn awaitt(self: *Buffer) !*Buffer { for (self._shards.slice()) |*shard| { try shard.awaitt(self._api); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 1fd6f665..6a84da88 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -130,6 +130,19 @@ pub const Client = opaque { } }; +pub const DeviceX = opaque { + pub fn getMemoryByKind(self: *const Device, api: *const Api, kind: pjrt.MemoryKind) ?*Memory { + var mem: ?*Memory = null; + const memories = self.inner().addressableMemories(api); + for (memories) |m| { + if (mem.kind(api) == kind) { + mem = m; + } + } + return mem; + } +}; + pub const Buffer = opaque { pub const inner = InnerMixin(pjrt.Buffer).inner; @@ -302,3 +315,11 @@ pub const LoadedExecutable = opaque { return try self.inner().getExecutable(api); } }; + +pub const AsyncHostToDeviceTransferManager = opaque { + const inner = InnerMixin(pjrt.AsyncHostToDeviceTransferManager).inner; + + pub fn deinit(self: *AsyncHostToDeviceTransferManager, api: *const Api) void { + self.inner().deinit(api); + } +}; diff --git a/zml/platform.zig b/zml/platform.zig index 67386303..2cd6b0f5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -7,6 +7,7 @@ const stdx = @import("stdx"); const meta = @import("meta.zig"); const module = @import("module.zig"); const pjrt = @import("pjrtx.zig"); +const Shape = @import("shape.zig").Shape; const log = std.log.scoped(.zml); @@ -87,6 +88,96 @@ pub const Platform = struct { } }; +// struct PJRT_AsyncHostToDeviceTransferManager_TransferData_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// int buffer_index; +// const void* data; +// int64_t offset; +// int64_t transfer_size; +// bool is_last_transfer; +// PJRT_Event* done_with_h2d_transfer; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_RetrieveBuffer_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// int buffer_index; +// PJRT_Buffer* buffer_out; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_Device_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// PJRT_Device* device_out; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_BufferCount_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// size_t buffer_count; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_BufferSize_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// int buffer_index; +// size_t buffer_size; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_AddMetadata_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// const PJRT_NamedValue* transfer_metadata; +// size_t num_metadata; +// }; + +// struct PJRT_Client_CreateBuffersForAsyncHostToDevice_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_Client* client; +// PJRT_ShapeSpec* shape_specs; +// size_t num_shape_specs; +// PJRT_Buffer_MemoryLayout** device_layouts; // optional +// size_t num_device_layouts; +// PJRT_Memory* memory; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; // out +// }; + +pub const TransferManager = struct { + pjrt_client: *pjrt.Client, + pjrt_api: *pjrt.Api, + pjrt_transfer_manager: []*pjrt.AsyncHostToDeviceTransferManager, + shape_specs: []const Shape, + memory: *pjrt.Memory, + + pub fn init(platform: Platform, memory_kind: pjrt.Memory.Kind, shapes: []Shape) !TransferManager { + const device = platform.getDevices()[0]; + const memory = device.getMemoryByKind(memory_kind); + if (memory == null) { + stdx.debug.panic("Device {s} doesn't have memory of kind {s}", .{ device.getName(), @tagName(memory_kind) }); + } + + return .{ + .pjrt_client = platform.pjrt_client, + .pjrt_api = platform.pjrt_api, + .pjrt_transfer_manager = try pjrt.Client.createBuffersForAsyncHostToDevice(platform.pjrt_client, memory, null), + .shape_specs = shapes, + .memory = memory, + }; + } + + pub fn deinit(self: *TransferManager) void { + pjrt.AsyncHostToDeviceTransferManager.deinit(self.pjrt_transfer_manager, self.pjrt_api); + } +}; + const _CreateOptions = struct { // XLA CPU client doesn't read options // https://github.com/openxla/xla/blob/42496a28c374bd35f493cc5dbde74805407245dc/xla/pjrt/c/pjrt_c_api_cpu_internal.cc#L33-L46 From 00ac82ab556bf54b00b2c0c456f79e90d789853e Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 31 Jan 2025 12:52:56 +0000 Subject: [PATCH 20/21] dataInMemory --- zml/buffer.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index abe33241..af00ee24 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -162,10 +162,10 @@ pub const Buffer = struct { pub fn dataInMemory(self: Buffer) ![]const u8 { const shard_buffer = self._shards.get(0).buffer; - const memoryDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); - _ = memoryDataPointer; // autofix - stdx.debug.assert(self._shards.len == 1, "TODO: support sharded Buffer -> Host transfer", .{}); - return try self._shards.get(0).buffer.dataInMemory(self._api); + const opaqueDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); + const sizeInbytes = try shard_buffer.getOnDeviceSizeInBytes(self._api); + const data = @as([*]const u8, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + return data[0..@intCast(sizeInbytes)]; } pub fn awaitt(self: *Buffer) !*Buffer { From 69e6f3f17619c9478330266c54d9df330b9f96da Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 3 Feb 2025 14:30:57 +0000 Subject: [PATCH 21/21] tensor output memory --- mlir/dialects/stablehlo.zig | 17 ++++++++++++++ pjrt/pjrt.zig | 17 ++++++++------ runtimes/cuda/cuda.zig | 3 ++- zml/aio.zig | 15 ++++--------- zml/buffer.zig | 41 ++++++++++++++------------------- zml/exe.zig | 2 -- zml/module.zig | 45 +++++++++++++++++++++++++++---------- zml/pjrtx.zig | 25 +-------------------- zml/tensor.zig | 29 ++++++++++++++++++++++++ 9 files changed, 113 insertions(+), 81 deletions(-) diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index a5c59cb6..39074795 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -780,6 +780,23 @@ pub fn sharding(ctx: mlir.Context, inputs: []const mlir.Value, sharding_spec: ml }); } +pub fn annotate_device_placement(ctx: mlir.Context, inputs: []const mlir.Value, memory_kind: mlir.StringAttribute, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { + const frontend_attributes = mlir.DictionaryAttribute.init(ctx, &.{mlir.NamedAttribute.init(mlir.Identifier.get(ctx, "_xla_buffer_placement"), memory_kind.asAttr())}).asAttr(); + return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ + .operands = inputs, + .results = res_types, + .attributes = &.{ + .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, 1).asAttr() }, + .{ "call_target_name", mlir.StringAttribute.init(ctx, "annotate_device_placement").asAttr() }, + .{ "has_side_effect", mlir.BoolAttribute.init(ctx, true).asAttr() }, + .{ "backend_config", mlir.StringAttribute.init(ctx, &.{}).asAttr() }, + .{ "output_operand_aliases", mlir.ArrayAttribute.init(ctx, &.{}).asAttr() }, + .{ "mhlo.frontend_attributes", frontend_attributes }, + }, + .location = location, + }); +} + pub const DotDimensionNumbersAttribute = struct { _inner: c.MlirAttribute, diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index bf8f99e3..7b100bef 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -324,7 +324,7 @@ pub const Client = opaque { byte_strides: ?[]const i64, device: *const Device, host_buffer_semantics: HostBufferSemantics, - memory: *const Memory, + memory: ?*const Memory, }; pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { @@ -338,7 +338,7 @@ pub const Client = opaque { .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), .device = @ptrCast(@constCast(args.device)), - // .memory = @ptrCast(@constCast(args.memory)), + .memory = if (args.memory) |m| @ptrCast(@constCast(m)) else null, .device_layout = null, // TODO .done_with_host_buffer = null, .buffer = null, @@ -440,18 +440,19 @@ pub const Device = opaque { .device = self.inner(), }) catch unreachable; if (ret.memories) |memories| { - return @ptrCast(memories[0..ret.num_memories]); + return @ptrCast(@constCast(memories[0..ret.num_memories])); } return &.{}; } pub fn getMemoryByKind(self: *const Device, api: *const Api, kind: Memory.Kind) ?*const Memory { const memories = self.addressableMemories(api); - return for (memories) |m| blk: { + for (memories) |m| { if (m.kind(api) == kind) { - break :blk m; + return m; } - }; + } + return null; } }; @@ -871,6 +872,8 @@ pub const Event = opaque { }; pub const Memory = opaque { + // enum class MemoryType { kDevice = 0, kUnified, kCollective, kHost = 5 }; + // from xla/stream_executor/stream_executor.h enum class MemoryType pub const Kind = enum { device, pinned_host, @@ -894,7 +897,7 @@ pub const Memory = opaque { return std.meta.stringToEnum(Kind, kind_[0..ret.kind_size]) orelse unreachable; } - pub fn kindId(self: *const Memory, api: *const Api) usize { + pub fn kindId(self: *const Memory, api: *const Api) i32 { const ret = api.call(.PJRT_Memory_Kind_Id, .{ .memory = self.inner(), }) catch unreachable; diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index 38a8aa6d..79aa477e 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -62,8 +62,9 @@ pub fn load() !*const pjrt.Api { log.warn("Detected {s} in LD_LIBRARY_PATH. This can lead to undefined behaviors and crashes", .{nvidiaLibsPath}); } - _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB", "48", 1); + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB", "2", 1); _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_PREALLOCATE", "true", 1); + _ = c.setenv("CUDA_VISIBLE_DEVICES", "0", 1); // CUDA path has to be set _before_ loading the PJRT plugin. // See https://github.com/openxla/xla/issues/21428 diff --git a/zml/aio.zig b/zml/aio.zig index 164d3c81..dd6b8f66 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -547,19 +547,12 @@ pub fn loadModelBuffersWithPrefix( // We copy the shape, and let visitStructAndLoadBuffer write the other fields. // to write them just afterward. var res: zml.Bufferized(Model) = undefined; - const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device - var selected_mem: *const zml.pjrt.Memory = undefined; - for (memories) |mem| { - if (mem.kind(platform.pjrt_api) == zml.pjrt.Memory.Kind.device) { - selected_mem = mem; - } - } try zml.meta.mapAlloc(struct { - pub fn initBuffer(mem: *const zml.pjrt.Memory, tensor: zml.Tensor) zml.Buffer { - return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined, ._memory = mem }; + pub fn initBuffer(_: void, tensor: zml.Tensor) zml.Buffer { + return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined }; } - }.initBuffer, allocator, selected_mem, model, &res); + }.initBuffer, allocator, {}, model, &res); var prefix_builder: PrefixBuilder = .{}; try prefix_builder.push(allocator, prefix); @@ -597,7 +590,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from2(platform, host_buffer.data, obj._shape); + obj.* = try zml.Buffer.from(platform, host_buffer); _ = try obj.awaitt(); } else { return error.BufferNotFound; diff --git a/zml/buffer.zig b/zml/buffer.zig index af00ee24..ba937f45 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -47,11 +47,11 @@ pub const Buffer = struct { _shape: Shape, _api: *const pjrt.Api, - _memory: *const pjrt.Memory, _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = std.BoundedArray(Shard, MAX_NUM_SHARDS); + pub const Memory = pjrt.Memory; /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { @@ -70,19 +70,11 @@ pub const Buffer = struct { const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); const devices = platform.getDevices(); - const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device - var selected_mem: *const pjrt.Memory = undefined; - for (memories) |mem| { - if (mem.kind(platform.pjrt_api) == pjrt.Memory.Kind.device) { - selected_mem = mem; - } - } var res: Buffer = .{ ._api = platform.pjrt_api, ._shape = host_buffer.shape(), ._shards = .{}, - ._memory = selected_mem, }; for (0..n_partitions) |i| { @@ -97,8 +89,8 @@ pub const Buffer = struct { .dims = buf.shape().dims(), .byte_strides = byte_strides, .device = devices[i], + .memory = null, .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .memory = res._memory, }); res._shards.appendAssumeCapacity(.{ .buffer = pjrt_buffer, @@ -109,7 +101,7 @@ pub const Buffer = struct { return res; } - pub fn from2(platform: Platform, buffer_: []const u8, shape_: Shape) !Buffer { + pub fn fromToMemory(platform: Platform, buffer_: []const u8, shape_: Shape, memory_kind: MemoryKind) !Buffer { // We shard only on the first axis so that the chunks are still contiguous. // TODO: support more advanced sharding specs stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); @@ -123,19 +115,11 @@ pub const Buffer = struct { const byte_strides = shape_.computeStrides().constSlice(); const devices = platform.getDevices(); - const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device - var selected_mem: *const pjrt.Memory = undefined; - for (memories) |mem| { - if (mem.kind(platform.pjrt_api) == pjrt.Memory.Kind.device) { - selected_mem = mem; - } - } var res: Buffer = .{ ._api = platform.pjrt_api, ._shape = shape_, ._shards = .{}, - ._memory = selected_mem, }; for (0..n_partitions) |i| { @@ -146,11 +130,9 @@ pub const Buffer = struct { .byte_strides = byte_strides, .device = devices[i], .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .memory = res._memory, + .memory = devices[i].getMemoryByKind(platform.pjrt_api, memory_kind), }); - // try pjrt_buffer.increaseExternalReferenceCount(platform.pjrt_api); - res._shards.appendAssumeCapacity(.{ .buffer = pjrt_buffer, .ready_event = event, @@ -160,12 +142,24 @@ pub const Buffer = struct { return res; } + pub fn getMemory(self: Buffer) *const Memory { + return self._shards.get(0).buffer.getMemory(self._api); + } + pub fn dataInMemory(self: Buffer) ![]const u8 { const shard_buffer = self._shards.get(0).buffer; const opaqueDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); const sizeInbytes = try shard_buffer.getOnDeviceSizeInBytes(self._api); const data = @as([*]const u8, @ptrFromInt(@intFromPtr(opaqueDataPointer))); - return data[0..@intCast(sizeInbytes)]; + const end: usize = @intCast(sizeInbytes); + return data[0..end]; + } + + pub fn getValueFromDataInMemory(self: Buffer, T: type) !T { + stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); + const data = try self.dataInMemory(); + const value = std.mem.bytesAsValue(T, @constCast(data)); + return value.*; } pub fn awaitt(self: *Buffer) !*Buffer { @@ -380,7 +374,6 @@ pub const Buffer = struct { ._api = self._api, ._shape = self._shape, ._shards = shards, - ._memory = selected_mem, }; } diff --git a/zml/exe.zig b/zml/exe.zig index ba63f238..c2c219a8 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -246,13 +246,11 @@ pub const BaseExe = struct { .ready = false, }); } - const output_mem = self.output_per_device[0][0].getMemory(self.platform.pjrt_api); return .{ ._shape = self.result_shapes[i], ._api = self.platform.pjrt_api, ._shards = shards, - ._memory = output_mem, // todo }; } }; diff --git a/zml/module.zig b/zml/module.zig index a003896d..2ade947a 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -398,6 +398,7 @@ pub const CompilationContext = struct { const fn_res_types = try res_allocator.alloc(mlir.Type, out_tensor_count); const fn_res_shapes = try res_allocator.alloc(Shape, out_tensor_count); const fn_res_donations = try res_allocator.alloc(Tensor._Donation, out_tensor_count); + const fn_res_output_memory_kind = try res_allocator.alloc(Tensor._MemoryKind, out_tensor_count); var fn_body = self.openBlock(.hermetic, input_types, locations) catch unreachable; { defer self.closeBlock(fn_body); @@ -413,7 +414,7 @@ pub const CompilationContext = struct { }; var fn_res_values: [out_tensor_count]mlir.Value = undefined; - self.extractValuesAndTypes(&fn_res, &fn_res_values, fn_res_types, fn_res_shapes, fn_res_donations); + self.extractValuesAndTypes(&fn_res, &fn_res_values, fn_res_types, fn_res_shapes, fn_res_donations, fn_res_output_memory_kind); const fn_ret = dialect.func.return_(mlir_ctx, &fn_res_values, loc); fn_body.appendOperationRecursive(fn_ret); @@ -427,11 +428,14 @@ pub const CompilationContext = struct { if (opts.kind == .main) { self.addDonationsAttributes(arg_attrs, fn_res_donations); + self.addOutputMemoryKindAttributes(res_attrs, fn_res_output_memory_kind); if (self._platform.sharding().num_partitions > 1) { self.addShardingAttributes(arg_attrs, res_attrs, input_shapes.items, fn_res_shapes); } } + std.debug.print("res_attrs : {any}", .{res_attrs}); + const mlir_fn = dialect.func.func(self.mlirCtx(), .{ .sym_name = opts.name, .args = input_types, @@ -463,6 +467,18 @@ pub const CompilationContext = struct { }; } + fn addOutputMemoryKindAttributes(self: CompilationContext, attributes: []AttributeList, output_memory_kind: []const Tensor._MemoryKind) void { + const mlir_ctx = self.mlirCtx(); + for (attributes, output_memory_kind) |*attr, memory_kind| { + if (memory_kind) |mk| { + attr.appendAssumeCapacity(mlir.NamedAttribute.init( + mlir.Identifier.get(mlir_ctx, "mhlo.memory_kind"), + mlir.StringAttribute.init(mlir_ctx, @tagName(mk)).as(mlir.Attribute).?, + )); + } + } + } + /// Given a list of donations mapping output buffers to input buffers, /// generate donation attribute for each `n_args` input argument. fn addDonationsAttributes(self: CompilationContext, attributes: []AttributeList, donations: []const Tensor._Donation) void { @@ -760,7 +776,7 @@ pub const CompilationContext = struct { } /// Visit the given struct and extract the mlir.Value and mlir.Type associated with each tensor found. - pub fn extractValuesAndTypes(self: *const CompilationContext, v: anytype, values: []mlir.Value, types: []mlir.Type, shapes: []Shape, donations: []Tensor._Donation) void { + pub fn extractValuesAndTypes(self: *const CompilationContext, v: anytype, values: []mlir.Value, types: []mlir.Type, shapes: []Shape, donations: []Tensor._Donation, output_memory_kind: []Tensor._MemoryKind) void { std.debug.assert(values.len == types.len); const LocalContext = struct { self: *const CompilationContext, @@ -769,8 +785,9 @@ pub const CompilationContext = struct { types: []mlir.Type, shapes: []Shape, donations: []Tensor._Donation, + output_memory_kind: []Tensor._MemoryKind, }; - var context = LocalContext{ .self = self, .values = values, .types = types, .shapes = shapes, .donations = donations }; + var context = LocalContext{ .self = self, .values = values, .types = types, .shapes = shapes, .donations = donations, .output_memory_kind = output_memory_kind }; meta.visit((struct { fn cb(ctx: *LocalContext, tensor: *const Tensor) void { const value, const donation = ctx.self.getValueAndDonation(tensor.*); @@ -778,6 +795,7 @@ pub const CompilationContext = struct { ctx.types[ctx.index] = value.getType(); ctx.shapes[ctx.index] = tensor._shape; ctx.donations[ctx.index] = donation; + ctx.output_memory_kind[ctx.index] = tensor._output_memory_kind; ctx.index += 1; } }).cb, &context, v); @@ -905,15 +923,18 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables setFlag(&options, "xla_gpu_enable_triton_gemm", false); setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); - // setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); - // setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); - // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); - // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); - // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); - // setFlag(&options, "xla_gpu_enable_custom_fusions", true); - // setFlag(&options, "xla_gpu_enable_dynamic_slice_fusion", true); - // setFlag(&options, "xla_gpu_enable_while_loop_double_buffering", true); - // setFlag(&options, "xla_gpu_use_runtime_fusion", true); + setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); + setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); + // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); + // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); + // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); + // setFlag(&options, "xla_gpu_enable_custom_fusions", true); + // setFlag(&options, "xla_gpu_enable_dynamic_slice_fusion", true); + // setFlag(&options, "xla_gpu_enable_while_loop_double_buffering", true); + // setFlag(&options, "xla_gpu_use_runtime_fusion", true); + // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); + // setFlag(&options, "xla_gpu_enable_dynamic_slice_fusion", true); + // setFlag(&options, "xla_gpu_async_dot", true); }, .rocm => { // Disable Triton GEMM on ROCM. For some reason it's much, much slower when diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 6a84da88..6d470059 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -130,24 +130,10 @@ pub const Client = opaque { } }; -pub const DeviceX = opaque { - pub fn getMemoryByKind(self: *const Device, api: *const Api, kind: pjrt.MemoryKind) ?*Memory { - var mem: ?*Memory = null; - const memories = self.inner().addressableMemories(api); - for (memories) |m| { - if (mem.kind(api) == kind) { - mem = m; - } - } - return mem; - } -}; - pub const Buffer = opaque { pub const inner = InnerMixin(pjrt.Buffer).inner; pub fn deinit(self: *Buffer, api: *const Api) void { - // self.inner().decreaseExternalReferenceCount(api); self.inner().deinit(api); } @@ -240,28 +226,19 @@ pub const Event = opaque { defer self.deinit(api); if (self.isReady(api)) { - std.log.warn("Call isReady already: {any}", .{self}); return; } - var timer = try stdx.time.Timer.start(); - var ctx = struct { err: ?*pjrt.Error = null, event: asynk.threading.ResetEventSingle = .{}, - timer: *stdx.time.Timer, - }{ - .timer = &timer, - }; + }{}; - std.log.warn("Call onReady: {any}", .{self}); try self.inner().onReady(api, &(struct { fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.C) void { const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); - var timer_ = ctx_.timer; ctx_.err = err; ctx_.event.set(); - std.log.warn("Callback called: {}", .{timer_.read()}); } }.call), &ctx); ctx.event.wait(); diff --git a/zml/tensor.zig b/zml/tensor.zig index f2e37497..ada19137 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -40,9 +40,11 @@ pub const Tensor = struct { _shape: Shape, _id: _Id, _donation: _Donation = .no_buffer, + _output_memory_kind: _MemoryKind = undefined, pub const _Donation = union(enum) { no_buffer, input_buffer, arg: u16 }; pub const _Id = union(enum) { mlir: mlir.Value, buffer_id: u64, arg_id: u64 }; + pub const _MemoryKind = ?Buffer.Memory.Kind; pub const MAX_RANK = Shape.MAX_RANK; @@ -193,6 +195,33 @@ pub const Tensor = struct { }; } + pub fn toMemory(self: Tensor, kind: Buffer.Memory.Kind) Tensor { + return switch (self._id) { + .arg_id, .mlir => { + const ctx = self.getContext(); + var res = self; + res._output_memory_kind = kind; + + const memory_kind = mlir.StringAttribute.init(ctx.mlirCtx(), @tagName(kind)); + + const op = dialect.stablehlo.annotate_device_placement( + ctx.mlirCtx(), + &.{self.value()}, + memory_kind, + &.{self.value().getType()}, + ctx.mlirCtx().location(@src()), + ); + + return _result(res._shape, op.result(0)); + }, + .buffer_id => { + var res = self; + res._output_memory_kind = kind; + return res; + }, + }; + } + /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { var res = self;