From fe7206628bcb42aea485f74f77740ac53074b4df Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 31 Mar 2025 15:16:17 +0000 Subject: [PATCH 01/22] zml/buffer: resuscitate async buffers --- zml/aio.zig | 2 +- zml/buffer.zig | 77 +++++++++++++++++++++++++++++++++++--------------- zml/pjrtx.zig | 14 ++++----- 3 files changed, 61 insertions(+), 32 deletions(-) diff --git a/zml/aio.zig b/zml/aio.zig index 98299908..38f996f9 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -654,7 +654,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from(platform, buf_with_metadata); + obj.* = try zml.Buffer.from(platform, buf_with_metadata, .{}); } else { log.err("Buffer not found: {s}", .{prefix}); diff --git a/zml/buffer.zig b/zml/buffer.zig index e0829d39..624c2957 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -68,8 +68,12 @@ pub const Buffer = struct { pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const FromOptions = struct { + wait: bool = true, + }; + /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { + pub fn from(platform: Platform, host_buffer: HostBuffer, opts: FromOptions) !Buffer { var res: Buffer = .{ ._api = platform.pjrt_api, ._shape = host_buffer.shape(), @@ -90,7 +94,6 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { // If no sharding if found, the given buffer is replicated on all devices. @@ -99,29 +102,39 @@ pub const Buffer = struct { break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - pjrt.Client.BufferFromHostBufferArgs{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, pjrt.Client.BufferFromHostBufferArgs{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - frames.appendAssumeCapacity(frame); - } + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); res._shards.appendAssumeCapacity(pjrt_buffer); } + + if (opts.wait) { + res = try res.awaitt(); + } + return res; } + pub fn awaitt(self: Buffer) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.await_(self._api); + } + } + + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -139,20 +152,40 @@ pub const Buffer = struct { /// return a Buffer with the given dimensions. pub fn fromSlice(platform: Platform, dimz: anytype, s: anytype) !Buffer { const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); - return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); + return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s)), .{}); } /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromBytes(platform: Platform, sh: Shape, data: []const u8) !Buffer { - return from(platform, HostBuffer.fromBytes(sh, data)); + return from(platform, HostBuffer.fromBytes(sh, data), .{}); } /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { const host_buffer = HostBuffer.fromArray(&arr); - return try from(platform, host_buffer); + return try from(platform, host_buffer, .{}); + } + + /// Copies the given Zig slice to the accelerator memory and + /// return a Buffer with the given dimensions. + pub fn fromSliceOpts(platform: Platform, dimz: anytype, s: anytype, opts: FromOptions) !Buffer { + const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); + return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s)), opts); + } + + /// Copies the given Zig slice to the accelerator memory and + /// return a Buffer with the given dimensions. + pub fn fromBytesOpts(platform: Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { + return from(platform, HostBuffer.fromBytes(sh, data), opts); + } + + /// Copies the given Zig array to the accelerator memory and + /// return a Buffer using the array shape. + pub fn fromArrayOpts(platform: Platform, arr: anytype, opts: FromOptions) !Buffer { + const host_buffer = HostBuffer.fromArray(&arr); + return try from(platform, host_buffer, opts); } /// Creates a Buffer with a single element. @@ -185,7 +218,7 @@ pub const Buffer = struct { ._strides = [1]i64{0} ** Shape.MAX_RANK, .data = x.constSlice(), }; - return try from(platform, host_buffer); + return try from(platform, host_buffer, .{}); } // To speed up copies, duplicate the scalar value into a vector, @@ -208,7 +241,7 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, .data = &bytes }; - return try from(platform, host_buffer); + return try from(platform, host_buffer, .{}); } test constant { diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index bd094ead..de1bf570 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -57,13 +57,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return @ptrCast(buffer); + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_)}; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -265,13 +261,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 2734184b3d79585fdc85ba593a6403d90e70e516 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 31 Mar 2025 15:16:52 +0000 Subject: [PATCH 02/22] zml/aio: implement awaitAll --- zml/aio.zig | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zml/aio.zig b/zml/aio.zig index 38f996f9..cfcbcd27 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -632,8 +632,11 @@ fn findSimilarBufferKeys(original_key: []const u8, store: BufferStore, temp_allo /// deinit all buffers in the given struct pub fn awaitAll(buffers: anytype) !void { - // TODO: implement once we have async buffers. - _ = buffers; + zml.meta.visit((struct { + fn cb(_: void, buffer: *zml.Buffer) void { + buffer.* = buffer.awaitt() catch unreachable; + } + }).cb, {}, buffers); } fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, buffer_store: BufferStore, obj: anytype, platform: zml.Platform) !void { From 4d226f4752ac0faa848d4931c8dcd9da04700a40 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 31 Mar 2025 15:17:06 +0000 Subject: [PATCH 03/22] zml/exe: disable execute awaiting --- zml/exe.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index 946410c6..c59f8e4a 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -211,11 +211,11 @@ pub const BaseExe = struct { .non_donatable_input_indices = &.{}, }) catch unreachable; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + //for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + //} } pub fn serialize(self: BaseExe, writer: anytype) !void { From 2a0fdd3c1901c4f443add1aa1eef70b151f0cee2 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 10 Apr 2025 08:58:43 +0000 Subject: [PATCH 04/22] mlir/dialects/stablehlo: added defaults for (operand|result)_layouts --- mlir/dialects/stablehlo.zig | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index 63283219..24466ddf 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -786,12 +786,32 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); } + const MINOR_TO_MAJOR = blk: { + const MAX_RANK = 8; + var ret: [MAX_RANK]usize = undefined; + for (0..MAX_RANK) |i| { + ret[i] = @intCast(MAX_RANK - i - 1); + } + break :blk ret; + }; + if (opts.operand_layouts) |layouts| { var operand_layouts: std.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; for (layouts) |ol| { operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); } attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); + } else { + const operand_layouts = blk: { + var ret: std.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; + for (inputs) |input| { + const ranked_type = input.getType().as(mlir.RankedTensorType); + const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; + ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); + } + break :blk ret; + }; + attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); } if (opts.result_layouts) |layouts| { @@ -800,6 +820,17 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); } attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); + } else { + const result_layouts = blk: { + var ret: std.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; + for (res_types) |t| { + const ranked_t = t.as(mlir.RankedTensorType); + const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; + ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); + } + break :blk ret; + }; + attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); } attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); From 62f664ac56eea2012293f108b2df18cb7ffb4d3e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 10 Apr 2025 08:59:34 +0000 Subject: [PATCH 05/22] zml/nn: bind paged sdpa --- zml/nn.zig | 2 +- zml/nn/cuda.zig | 151 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index c3120d26..b5560f09 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,7 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const cuda = @import("nn/cuda.zig"); +pub const cuda = @import("nn/cuda.zig"); const helpers = @import("helpers.zig"); const meta = @import("meta.zig"); const ops = @import("ops.zig"); diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index 8652f9bc..e087bc61 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -138,3 +138,154 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const result = Tensor._result(q.shape(), op.result(0)).transpose(q_.shape()); return result; } + +pub const SdpaPagedOpts = struct { + attn_mask: ?Tensor = null, + scale: ?Tensor = null, + max_seq_len_kv: usize, + max_input_len: usize, +}; + +pub fn layout_constraint(input: Tensor, output_shape: Shape) Tensor { + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + &.{input.value()}, + .{ + .call_target_name = "LayoutConstraint", + .api_version = 2, + .has_side_effect = false, + .output_operand_aliases = &.{}, + }, + &.{ + mlir.ext.mlirType(mlir_ctx, output_shape), + //mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type).?).asType(), + }, + loc, + ); + const result = Tensor._result(output_shape, op.result(0)); + return result; +} + +pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, sequence_length_kv: Tensor, page_table: Tensor, seq_offset: Tensor, opts: SdpaPagedOpts) Tensor { + const ctx = CompilationContext.current(); + //var q = q_; + //var k = k_; + //var v = v_; + //const q = layout_constraint(q_, q_.shape().transpose(.{ .b, .h, .q, .hd })); + //var k = layout_constraint(k_, k_.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + //const v = layout_constraint(v_, v_.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + const q = q_.transpose(.{ .b, .h, .q, .hd }); + var k = k_.transpose(.{ .page, .h, .k_chunk, .hd }); + const v = v_.transpose(.{ .page, .h, .k_chunk, .hd }); + + const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); + const kernel_scale = if (opts.scale) |_| 1.0 else sqrtHeadDim; + if (opts.scale) |s| { + k = k.mul(s); + } + + var buffer: [4096]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buffer); + const allocator = fba.allocator(); + + const backend_config = + std.fmt.allocPrintZ( + allocator, + \\{{ + \\ "operation_queue_id":"0", + \\ "wait_on_operation_queues":[], + \\ "cudnn_fmha_backend_config": {{ + \\ "algorithm": {{ + \\ "algo_id": "0", + \\ "math_type": "TENSOR_OP_MATH", + \\ "tuning_knobs": {{"17": "1", "24": "0"}}, + \\ "is_cudnn_frontend": true, + \\ "workspace_size": "0", + \\ }}, + \\ "fmha_scale": {}, + \\ "dropout_rate": 0, + \\ "intermediate_tensor_shape": {{ + \\ "element_type": "{s}", + \\ "dimensions": ["{}", "{}", "{}", "{}"], + \\ "tuple_shapes": [], + \\ "layout": {{ + \\ "dim_level_types": [], + \\ "dim_unique": [], + \\ "dim_ordered": [], + \\ "minor_to_major": ["3", "2", "1", "0"], + \\ "tiles": [], + \\ "element_size_in_bits": "0", + \\ "memory_space": "0", + \\ "index_primitive_type": "PRIMITIVE_TYPE_INVALID", + \\ "pointer_primitive_type": "PRIMITIVE_TYPE_INVALID", + \\ "dynamic_shape_metadata_prefix_bytes": "0", + \\ }}, + \\ "is_dynamic_dimension": [false, false, false, false], + \\ }}, + \\ "seed": 0, + \\ "is_flash_attention": true, + \\ "mask_type": "NO_MASK", + \\ "bmm1_dot_dimension_numbers": {{ + \\ "lhs_contracting_dimensions": ["3"], + \\ "rhs_contracting_dimensions": ["3"], + \\ "lhs_batch_dimensions": ["0", "1"], + \\ "rhs_batch_dimensions": ["0", "1"], + \\ }}, + \\ "bmm2_dot_dimension_numbers": {{ + \\ "lhs_contracting_dimensions": ["3"], + \\ "rhs_contracting_dimensions": ["2"], + \\ "lhs_batch_dimensions": ["0", "1"], + \\ "rhs_batch_dimensions": ["0", "1"], + \\ }}, + \\ "max_sequence_length_kv": {}, + \\ "max_input_len": {}, + \\ }} + \\}} + , + .{ + kernel_scale, + elementTypeFromDataType(q.dtype()), + q.dim(.b), + q.dim(.h), + q.dim(.q), + k.dim(.k_chunk), + opts.max_seq_len_kv, + opts.max_input_len, + }, + ) catch unreachable; + + var bias = Tensor.constant(Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .k = opts.max_seq_len_kv }, q.dtype()), Data.init(q.dtype(), 0)); + + if (opts.attn_mask) |attn_mask| { + bias = bias.add(attn_mask.broad(bias.shape())); + } + + //q = q_.transpose(.{ .b, .h, .q, .hd }); + //q = layout_constraint(q, q.shape()); + //k = layout_constraint(k, k.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + //v = layout_constraint(v, v.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + const output_shape = Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .hd = q.dim(.hd) }, q.dtype()); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + &.{ q.value(), k.value(), v.value(), sequence_length_q.value(), sequence_length_kv.value(), page_table.value(), page_table.value(), seq_offset.value() }, + .{ + .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", + .backend_config = mlir.Attribute.string(mlir_ctx, backend_config), + .api_version = .status_returning, + .has_side_effect = false, + .output_operand_aliases = &.{}, + }, + &.{ + mlir.ext.mlirType(mlir_ctx, output_shape), + mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type)).as(mlir.Type), + }, + loc, + ); + const result = Tensor._result(output_shape, op.result(0)).transpose(q_.shape()); + return result; +} From 2d86823127d13369479567fa15f14105cd1f1a3e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 10 Apr 2025 19:37:20 +0000 Subject: [PATCH 06/22] wip --- pjrt/pjrt.zig | 4 ++-- zml/buffer.zig | 10 ++++++++++ zml/ops.zig | 4 ++-- zml/pjrtx.zig | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9dbd5aac..3ded8524 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,7 +31,7 @@ fn pjrtStructSize(comptime T: type) usize { return @field(c, typedef_name ++ "_STRUCT_SIZE"); } -inline fn pjrtStruct(v: anytype) @TypeOf(v) { +pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { var ret = v; ret.struct_size = pjrtStructSize(@TypeOf(v)); return ret; @@ -925,7 +925,7 @@ pub const Memory = opaque { pub fn kind(self: *const Memory, api: *const Api) Kind { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable[0..ret.kind_size]; + const kind_ = ret.kind[0..ret.kind_size]; return std.meta.stringToEnum(Kind, kind_) orelse unreachable; } diff --git a/zml/buffer.zig b/zml/buffer.zig index 624c2957..c6277fa9 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -394,6 +394,16 @@ pub const Buffer = struct { if (self._shards.len == 1) return false; return @reduce(.Or, self._shape._sharding_info); } + + pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { + var new_shards: Buffer.Shards = .{}; + for (self._shards.slice()) |shard| { + const new_shard = try shard.copyToMemory(self._api, memory); + new_shards.appendAssumeCapacity(new_shard); + } + + return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; + } }; pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { diff --git a/zml/ops.zig b/zml/ops.zig index 1765888a..efe5d36c 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -22,8 +22,8 @@ const ShapeOf = @import("tensor.zig").ShapeOf; const Tensor = @import("tensor.zig").Tensor; const EnumLiteral = @TypeOf(.enum_literal); -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; +pub const dialect = struct { + pub const stablehlo = @import("mlir/dialects").stablehlo; }; const log = std.log.scoped(.@"zml/tensor"); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index de1bf570..b2b04c1d 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -116,7 +116,7 @@ pub const Client = opaque { return self.inner().addressableMemories(api); } - pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*Memory { + pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { for (self.addressableMemories(api)) |mem| { if (mem.kind(api) == kind) { return mem; @@ -178,7 +178,7 @@ pub const Buffer = opaque { } pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(self.inner().copyToMemory(api, memory_)); + return @ptrCast(try self.inner().copyToMemory(api, memory_)); } pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { From ddd170578dd5744c6e253a8209d48762d77fcc10 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 18 Apr 2025 14:29:19 +0000 Subject: [PATCH 07/22] zml/tensor.zig: fix returned value in Tensor.toMemory --- zml/tensor.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index b248ea20..79c2f2c9 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -206,8 +206,6 @@ pub const Tensor = struct { const ctx = self.getContext(); const mlir_ctx = ctx.mlirCtx(); if (ctx.target() == .cpu) return self; - var res = self; - res._output_memory_kind = kind; const memory_kind = @tagName(kind.toPjrtMemory()); @@ -223,7 +221,9 @@ pub const Tensor = struct { .api_version = .original, }, &.{self.value().getType()}, mlir_ctx.location(@src())); - return _result(res._shape, op.result(0)); + var res = _result(self._shape, op.result(0)); + res._output_memory_kind = kind; + return res; }, .buffer_id => { var res = self; From 3eef25e8cb3bd5900dd5b79617b8007c5099d1fa Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 18 Apr 2025 14:30:14 +0000 Subject: [PATCH 08/22] pjrt: bind PJRT_Device_AddressableMemories --- pjrt/pjrt.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 3ded8524..f9a13e31 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -502,6 +502,13 @@ pub const Device = opaque { }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) ApiError![]const *Memory { + const ret = try api.call(.PJRT_Device_AddressableMemories, .{ + .device = self.inner(), + }); + return @ptrCast(ret.memories[0..ret.num_memories]); + } }; pub const DeviceDescription = opaque { From ee263eb55e189559c54afc2fad16fa5b67f8d377 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 23 Apr 2025 10:52:17 +0200 Subject: [PATCH 09/22] zml/buffer.zig: add a way to specify memory kind for buffer creation --- zml/buffer.zig | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index c6277fa9..0b86ae6a 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -70,6 +70,7 @@ pub const Buffer = struct { pub const FromOptions = struct { wait: bool = true, + memory: ?pjrt.Memory.Kind = null, }; /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -102,14 +103,25 @@ pub const Buffer = struct { break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, pjrt.Client.BufferFromHostBufferArgs{ + var args = pjrt.Client.BufferFromHostBufferArgs{ .data = buf.data, .buffer_type = buffer_type, .dims = buf.shape().dims(), .byte_strides = byte_strides, - .device = devices[i], .host_buffer_semantics = .ImmutableUntilTransferCompletes, - }); + }; + if (opts.memory) |memory_kind| { + const memories = try devices[i].addressableMemories(platform.pjrt_api); + const memory = for (memories) |m| { + const kind = m.kind(platform.pjrt_api); + if (kind == memory_kind) break m; + } else return error.NotFound; + args.memory = memory; + } else { + args.device = devices[i]; + } + + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); if (event) |ev| { ev.deinit(platform.pjrt_api); From ead9be07fb2edfc847574b7e39ab4dbb6b688ea5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 23 Apr 2025 10:52:55 +0200 Subject: [PATCH 10/22] runtimes/cuda: use local plugin --- runtimes/cuda/BUILD.bazel | 5 +++++ runtimes/cuda/libpjrt_cuda.BUILD.bazel | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index a7f8bf94..eebbcb12 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -34,6 +34,11 @@ zig_library( }), ) +exports_files( + ["libpjrt_cuda.so"], + visibility = ["//visibility:public"], +) + filegroup( name = "layers", srcs = [], diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index 8881a016..17318a49 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -25,7 +25,8 @@ copy_to_directory( cc_import( name = "libpjrt_cuda", data = [":sandbox"], - shared_library = "libpjrt_cuda.so", + shared_library = "@zml//runtimes/cuda:libpjrt_cuda.so", + #shared_library = "libpjrt_cuda.so", add_needed = ["libzmlxcuda.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", From 06cd5a7188d62b5bf9d041eee1da36e841724bc7 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 24 Apr 2025 17:29:59 +0000 Subject: [PATCH 11/22] zml/meta.zig: support union in map and mapAlloc --- zml/meta.zig | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/zml/meta.zig b/zml/meta.zig index 16680eb3..ead85d2d 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -59,6 +59,31 @@ pub fn MapType(From: type, To: type) type { .is_tuple = struct_infos.is_tuple, } }); }, + .@"union" => |union_info| { + const fields = union_info.fields; + var same: bool = true; + var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; + for (union_fields[0..], fields) |*union_field, field| { + const R = map(field.type); + if (R == field.type) { + union_field.* = field; + } else { + union_field.* = .{ + .name = field.name, + .type = R, + .alignment = @alignOf(R), + }; + same = false; + } + } + if (same) return T; + return @Type(.{ .@"union" = .{ + .layout = .auto, + .tag_type = union_info.tag_type, + .fields = union_fields[0..], + .decls = &.{}, + } }); + }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { .slice => if (ptr_info.is_const) @@ -144,6 +169,15 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam else => @field(to, field.name) = @field(from, field.name), } }, + .@"union" => |info| { + _ = info; // autofix + switch (from) { + inline else => |_, tag| { + to.* = @unionInit(ToStruct, @tagName(tag), undefined); + try mapAlloc(cb, allocator, ctx, @field(from, @tagName(tag)), &@field(to, @tagName(tag))); + }, + } + }, .array => for (from, to) |f, *t| { try mapAlloc(cb, allocator, ctx, f, t); }, @@ -174,7 +208,7 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } else { to.* = null; }, - .int, .float, .@"enum", .@"union" => to.* = from, + .int, .float, .@"enum" => to.* = from, else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), } } From bf7ce6034cfb6d3843605183df8dea0778cc6c32 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 28 Apr 2025 17:46:41 +0000 Subject: [PATCH 12/22] wip --- zml/pjrtx.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index b2b04c1d..089c985e 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const stdx = @import("stdx"); const asynk = @import("async"); const dialects = @import("mlir/dialects"); @@ -261,13 +262,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ + try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - }); + }}); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From de722633eb6d7896dca2fdc4b030189c22d97e17 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 29 Apr 2025 16:40:52 +0000 Subject: [PATCH 13/22] pjrt: wip wire command buffer compatibility --- pjrt/pjrt.zig | 11 +++++++++++ ...FFI-handler-registration-API-to-the-FFI-PjRt.patch | 9 +++++++-- zml/context.zig | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index f9a13e31..8ece2eef 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1164,17 +1164,27 @@ pub const NamedValue = extern struct { } }; +pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { + command_buffer_compatible = c.PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE, + _, +}; + // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub const CustomCallRegistry = extern struct { inner: *const c.PJRT_FFI_Register_Handler, + pub const RegisterFfiOptions = struct { + traits: RegisterHandlerTraits = @enumFromInt(0), + }; + pub fn registerFfi( self: *const CustomCallRegistry, api: *const Api, target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, + options: RegisterFfiOptions, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .api_version = 1, @@ -1183,6 +1193,7 @@ pub const CustomCallRegistry = extern struct { .handler = @ptrCast(@constCast(func)), .platform_name = platform_name.ptr, .platform_name_size = platform_name.len, + .traits = @intFromEnum(options.traits), }); const result = self.inner(&ret); if (result) |pjrt_c_error| { diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch index 58e3874d..bff748ec 100644 --- a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -31,10 +31,14 @@ diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_e index a33bd4aa9c..3309194538 100644 --- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h +++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); +@@ -66,12 +66,33 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); // Adds a user data to the execute context. typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); ++ typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ +struct PJRT_FFI_Register_Handler_Args { + size_t struct_size; + const char* target_name; @@ -43,8 +47,9 @@ index a33bd4aa9c..3309194538 100644 + void* handler; + const char* platform_name; + size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; +}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); + +// Registers an FFI call handler for a specific platform. +typedef PJRT_Error* PJRT_FFI_Register_Handler( diff --git a/zml/context.zig b/zml/context.zig index 03de62a3..874fea31 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -221,7 +221,7 @@ const CustomCall = struct { const registry = platform.pjrt_api.customCallRegistry(); if (registry) |reg| { - try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback); + try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback, .{}); } else { stdx.debug.panic("Registering custom calls failed", .{}); } From 0a3ca52705a809fda14087a0cdd1edaf39a6c5a6 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 29 Apr 2025 16:41:12 +0000 Subject: [PATCH 14/22] pjrtx: temporarily disable callBlocking for execute Now that we have command buffer, it's less of a problem --- zml/pjrtx.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 089c985e..f744622b 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -262,13 +262,20 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - }}); + }); + //try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + // .num_args = args.num_args, + // .arguments = @ptrCast(args.arguments), + // .results = @ptrCast(args.results), + // .events = @ptrCast(args.events), + // .non_donatable_input_indices = args.non_donatable_input_indices, + //}}); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 875b87feb601c551367b053a645eed7e875fbcfd Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 9 May 2025 16:28:20 +0000 Subject: [PATCH 15/22] zml/tensor.zig: Added `Tensor.zeroLike` --- zml/tensor.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zml/tensor.zig b/zml/tensor.zig index 79c2f2c9..8831985e 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1917,6 +1917,10 @@ pub const Tensor = struct { return _result(val.shape(), constant_op.result(0)); } + pub fn zeroLike(other: Tensor) Tensor { + return Tensor.constant(other.shape(), other.dtype().zero()); + } + /// Returns a Tensor containing the result of the outer product between the input Tensors. pub fn outer(self: Tensor, other: Tensor) Tensor { if (self.rank() + other.rank() == 1) { From 908a3a6be86f3ecf6732388ade07a2c134f66fab Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 20 May 2025 15:49:09 +0000 Subject: [PATCH 16/22] async: increase stack size to 2MB --- async/stack.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/async/stack.zig b/async/stack.zig index 1e08c8fb..b9703462 100644 --- a/async/stack.zig +++ b/async/stack.zig @@ -1,7 +1,7 @@ const std = @import("std"); const coro_base = @import("coro_base.zig"); -pub const stack_size = 1 * 1024 * 1024; +pub const stack_size = 2 * 1024 * 1024; pub const Stack = struct { pub const Data = struct { From a935d972a305208ddc13545de711b3ca47272c17 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 22 May 2025 17:18:14 +0000 Subject: [PATCH 17/22] wip --- zml/pjrtx.zig | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index f744622b..419b8542 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -262,20 +262,20 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ - .num_args = args.num_args, - .arguments = @ptrCast(args.arguments), - .results = @ptrCast(args.results), - .events = @ptrCast(args.events), - .non_donatable_input_indices = args.non_donatable_input_indices, - }); - //try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + //try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ // .num_args = args.num_args, // .arguments = @ptrCast(args.arguments), // .results = @ptrCast(args.results), // .events = @ptrCast(args.events), // .non_donatable_input_indices = args.non_donatable_input_indices, - //}}); + //}); + try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + .num_args = args.num_args, + .arguments = @ptrCast(args.arguments), + .results = @ptrCast(args.results), + .events = @ptrCast(args.events), + .non_donatable_input_indices = args.non_donatable_input_indices, + }}); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From d4047d3319cef948673fc6dab7caa4b59bebfda3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 May 2025 16:29:03 +0000 Subject: [PATCH 18/22] zml/ops.zig: added a generic customCall --- zml/ops.zig | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/zml/ops.zig b/zml/ops.zig index efe5d36c..c28b88a6 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1246,6 +1246,106 @@ fn scatterPrepareIndices( return Tensor.stack(indices.constSlice(), .last, .coord); } +fn TensorOrTensorArray(comptime T: type) type { + const type_info = @typeInfo(T); + return switch (type_info) { + .@"struct" => |struct_info| b: { + if (T == Tensor) break :b Tensor; + if (!struct_info.is_tuple) @compileError("Expected tuple"); + break :b if (struct_info.fields.len == 1) + Tensor + else + [struct_info.fields.len]Tensor; + }, + .array => |array_info| b: { + break :b if (array_info.len == 1) + Tensor + else + [array_info.len]Tensor; + }, + else => @compileError("Unsupported type: " ++ @typeName(T)), + }; +} + +pub const CustomCallOptions = struct { + output_operand_aliases: ?[]const i64 = null, +}; + +pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) TensorOrTensorArray(@TypeOf(outputs)) { + const ctx = module.CompilationContext.current(); + + var values: [inputs.len]mlir.Value = undefined; + ctx.extractValues(&inputs, &values); + + var res_types: [outputs.len]mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlir.ext.mlirType(ctx.mlirCtx(), output); + } + + const metadata_type_info = @typeInfo(@TypeOf(metadata)); + var metadata_attributes_tuple: [metadata_type_info.@"struct".fields.len]mlir.AttrTuple = undefined; + inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { + const attribute: mlir.Attribute = switch (@typeInfo(field.type)) { + .int, .comptime_int => mlir.Attribute.int(ctx.mlirCtx(), .i64, @intCast(@field(metadata, field.name))), + else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), + }; + metadata_attributes_tuple[i] = .{ field.name, attribute }; + } + + const backend_config = mlir.Attribute.dict(ctx.mlirCtx(), &(.{ + .{ "pjrt_api", mlir.Attribute.int(ctx.mlirCtx(), .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_api))) }, + .{ "pjrt_client", mlir.Attribute.int(ctx.mlirCtx(), .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_client))) }, + } ++ metadata_attributes_tuple)); + + var operands_layouts: [inputs.len][]const usize = undefined; + inline for (inputs, 0..) |input, i| { + operands_layouts[i] = minorToMajor(input.rank()); + } + + var results_layouts: [outputs.len][]const usize = undefined; + inline for (outputs, 0..) |output, i| { + results_layouts[i] = minorToMajor(output.rank()); + } + + const op = dialect.stablehlo.custom_call( + ctx.mlirCtx(), + &values, + .{ + .call_target_name = target_name, + .backend_config = backend_config, + .has_side_effect = true, + .api_version = .typed_ffi, + .operand_layouts = &operands_layouts, + .result_layouts = &results_layouts, + .output_operand_aliases = opts.output_operand_aliases orelse &.{}, + }, + &res_types, + ctx.mlirCtx().location(@src()), + ); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return switch (@typeInfo(@TypeOf(outputs))) { + .@"struct" => |struct_info| b: { + if (!struct_info.is_tuple) @compileError("Expected tuple"); + break :b if (struct_info.fields.len == 1) + outputs_[0] + else + outputs_; + }, + .array => |array_info| b: { + break :b if (array_info.len == 1) + outputs_[0] + else + outputs_; + }, + else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), + }; +} + inline fn toI64(values: anytype) []i64 { var res: [Tensor.MAX_RANK]i64 = undefined; for (values, 0..) |val, i| res[i] = @intCast(val); From 7edce02876d7a7255fef3067d01b9b4240270632 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 May 2025 16:35:48 +0000 Subject: [PATCH 19/22] zml/tensor.zig: allow tag in Tensor.dynamicSlice1d() --- zml/tensor.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index 8831985e..c5d008ef 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -3136,7 +3136,7 @@ pub const Tensor = struct { /// Slices the input Tensor along a specific axis, with a start offset known at runtime. /// Note: this doesn't support tagging, if you have tags, /// you should use `dynamicSlice` directly. - pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { + pub fn dynamicSlice1d(self: Tensor, axis_: anytype, slice_: DynSlice) Tensor { stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {}", .{slice_.start}); const a = self.axis(axis_); From 636c374ae13faca7f41c1bf55f4e4476c0632d66 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 9 May 2025 14:32:15 +0200 Subject: [PATCH 20/22] Fix llama3 rope scaling (#230) --- examples/llama/llama.zig | 19 ++-- examples/modernbert/modernbert.zig | 6 +- zml/nn.zig | 175 +++++++++++++++++++++++------ 3 files changed, 149 insertions(+), 51 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 96630359..8314a307 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -1,14 +1,12 @@ const std = @import("std"); +const testing = std.testing; + const stdx = @import("stdx"); const zml = @import("zml"); - -const testing = std.testing; const Buffer = zml.Buffer; const Tensor = zml.Tensor; const ShapeOf = zml.ShapeOf; -const gguf = zml.io.gguf; -const expectClose = zml.testing.expectClose; const log = std.log.scoped(.llama); /// Llama architecture, using huggingface transformers naming. @@ -27,6 +25,7 @@ pub const LlamaLM = struct { max_position_embeddings: usize, rms_norm_eps: f32, hf_rope_impl: bool = true, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, }; pub const Options = struct { @@ -48,8 +47,9 @@ pub const LlamaLM = struct { self.model.num_heads = @intCast(config.num_attention_heads); self.model.num_kv_heads = @intCast(config.num_key_value_heads); self.model.rope_opts = .{ - .impl = if (config.hf_rope_impl) .sequential else .interleaved, + .layout = if (config.hf_rope_impl) .sequential else .interleaved, .freq_base = config.rope_theta, + .scaling = config.rope_scaling, }; for (self.model.layers) |*layer| { layer.self_attn.num_heads = self.model.num_heads; @@ -131,7 +131,7 @@ pub const Llama = struct { num_heads: i64 = 32, num_kv_heads: i64 = 32, rope_opts: zml.nn.RopeOpts = .{ - .impl = .interleaved, + .layout = .interleaved, .freq_base = 10_000, }, @@ -221,12 +221,7 @@ const RmsNorm = struct { /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: Tensor) Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); - // upcast to improve precision - const xf32 = x.convert(.f32); - const mean = xf32.mul(xf32).mean(.d); - const rsqrt = Tensor.rsqrt(mean.addConstant(self.eps)).convert(x.dtype()); - const normalized = x.mul(rsqrt.broad(x.shape())); - + const normalized = zml.nn.rmsNorm(x, .d, self.eps); return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); } }; diff --git a/examples/modernbert/modernbert.zig b/examples/modernbert/modernbert.zig index 4a1182b6..831610c9 100644 --- a/examples/modernbert/modernbert.zig +++ b/examples/modernbert/modernbert.zig @@ -1,12 +1,12 @@ const std = @import("std"); -const log = std.log.scoped(.modernbert); const asynk = @import("async"); const stdx = @import("stdx"); const zml = @import("zml"); - const Tensor = zml.Tensor; +const log = std.log.scoped(.modernbert); + pub const ModernBertOptions = struct { num_attention_heads: i64, pad_token: u32, @@ -222,7 +222,7 @@ pub const ModernBertAttention = struct { // Layer 0, 3, 6, 9, 12 ... use global RoPE // Layer 1, 2, 4, 5, 7, 8, 10, 11 ... use local RoPE const rope_opts = zml.nn.RopeOpts{ - .impl = .sequential, + .layout = .sequential, .freq_base = if (self.is_global_attention) 160_000 else 10_000, }; diff --git a/zml/nn.zig b/zml/nn.zig index b5560f09..104d793f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1,21 +1,20 @@ //! Common layer definition and functions for Neural Networks (NN) const std = @import("std"); -const stdx = @import("stdx"); - -pub const cuda = @import("nn/cuda.zig"); -const helpers = @import("helpers.zig"); -const meta = @import("meta.zig"); -const ops = @import("ops.zig"); -const zml = @import("zml.zig"); - -const DataType = @import("dtype.zig").DataType; -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - const assert = std.debug.assert; -const log = std.log.scoped(.@"zml/tensor"); const testing = std.testing; +const stdx = @import("stdx"); + +const DataType = @import("dtype.zig").DataType; +const helpers = @import("helpers.zig"); +const meta = @import("meta.zig"); +const cuda = @import("nn/cuda.zig"); +const ops = @import("ops.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; +const zml = @import("zml.zig"); + +const log = std.log.scoped(.@"zml/tensor"); test { _ = cuda; std.testing.refAllDecls(@This()); @@ -108,6 +107,15 @@ pub const LayerNorm = struct { } }; +pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x.axis(axis); + // upcast to improve precision + const xf32 = x.convert(.f32); + const mean = xf32.mul(xf32).mean(ax); + const rsqrt = Tensor.rsqrt(mean.addConstant(eps)).convert(x.dtype()); + return x.mul(rsqrt.broad(x.shape())); +} + /// Center and scale by the variance. /// normalize(x, eps) = (x - mean(x)) / sqrt(var(x) + eps) /// Work on the last axis. @@ -142,13 +150,46 @@ test normalizeL2 { } pub const RopeOpts = struct { - /// There are two implementations corresponding to how to split `x` in real/imag parts. + layout: Layout = .sequential, + freq_base: f32 = 10_000, + scaling: Scaling = .default, + + /// There are two layouts corresponding to how to split `x` in real/imag parts. /// * Interleaved means that the real/imag of each scalar is contiguous. /// * Sequential means that you first read all real values then all imag values. - pub const Implementation = enum { interleaved, sequential }; + /// Typically HF models use sequential, while GGUF use interleaved. + pub const Layout = enum { interleaved, sequential }; - impl: Implementation, - freq_base: f32 = 10_000, + /// There are several ways to init the scaling aka "inv_freq" + pub const Scaling = union(enum) { + default: void, + custom: []const f32, + llama3: Llama3, + + pub const Llama3 = struct { + factor: f32, + high_freq_factor: f32, + low_freq_factor: f32, + original_max_position_embeddings: u32, + }; + + /// Read a Rope scaling config from HF config.json format. + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { + const content = try std.json.Value.jsonParse(allocator, source, options); + if (content != .object) return error.InvalidEnumTag; + + const obj = content.object; + const impl = obj.get("rope_type") orelse return error.MissingField; + if (impl != .string) return error.InvalidEnumTag; + if (std.mem.eql(u8, impl.string, "llama3")) { + // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, undefined, content, .{ .ignore_unknown_fields = true }) }; + } else { + log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); + return .{ .default = {} }; + } + } + }; }; /// Rotary position embedding modify queries and keys tensor before compute Q * K in self attention. @@ -170,10 +211,10 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={})", .{x}); break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); }; - const x_real, const x_imag = splitRealImg(x, opts.impl); + const x_real, const x_imag = splitRealImg(x, opts.layout); // compute sin and cos in f32 before downcasting to x type. - const inv_freq = invFreq(x.dim(.hd), opts.freq_base, .f32).withTags(.{.hd}); + const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); const cos = inv_freq_pos.cos().convert(x.dtype()).broad(x_real.shape()); const sin = inv_freq_pos.sin().convert(x.dtype()).broad(x_real.shape()); @@ -183,13 +224,13 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); // flatten last dimensions - return mergeRealImg(y_real, y_imag, opts.impl); + return mergeRealImg(y_real, y_imag, opts.layout); } -pub fn splitRealImg(x: Tensor, impl: RopeOpts.Implementation) [2]Tensor { +pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { const n = x.dim(-1); - return switch (impl) { + return switch (layout) { .sequential => .{ x.slice1d(-1, .{ .end = @divExact(n, 2) }), x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), @@ -201,8 +242,8 @@ pub fn splitRealImg(x: Tensor, impl: RopeOpts.Implementation) [2]Tensor { }; } -pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, impl: RopeOpts.Implementation) Tensor { - return switch (impl) { +pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Tensor { + return switch (layout) { .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), .interleaved => Tensor.concatenate(&.{ x_real.appendAxes(.{.interleaved_real_img}), @@ -212,21 +253,83 @@ pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, impl: RopeOpts.Implementatio } /// {exp( - n * ln(10_000) / N ) | n in [0..N] } -pub fn invFreq(N: i64, theta: f32, dtype: DataType) Tensor { - const freq = -@log(theta) / @as(f32, @floatFromInt(N)); - const range = Tensor.arange(.{ .start = 0, .end = N, .step = 2 }, dtype).scale(freq); - return range.exp(); +pub fn invFreq(N: i64, opts: RopeOpts) Tensor { + const allocator = zml.module.CompilationContext.current().allocator(); + const N_half: usize = @intCast(@divExact(N, 2)); + const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); + _invFreq(opts, inv_freq); + return zml.Tensor.constantTensor(.fromSlice(.{@divExact(N, 2)}, inv_freq)); +} + +fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { + const N = inv_freq.len; + // Default frequencies + for (0.., inv_freq) |n, *f| { + f.* = @exp(-@log(opts.freq_base) * stdx.math.divFloat(f32, n, N)); + } + + switch (opts.scaling) { + .default => {}, + .custom => { + stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {}, got {}", .{ N, opts.scaling.custom.len }); + @memcpy(inv_freq, opts.scaling.custom); + }, + .llama3 => |s| { + // https://arxiv.org/pdf/2309.16039 + // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. + // In Llama3 they used a higher base freq and also downscaled low frequencies. + std.debug.assert(s.low_freq_factor < s.high_freq_factor); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.high_freq_factor * (2 * std.math.pi) / M; + const f_low = s.low_freq_factor * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Linear interpolation for middle freq + const lerp: f64 = (inv_freq[n] - f_low) / (f_high - f_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, + } +} + +test invFreq { + // Llama 3.2-1B config + const llama_conf: RopeOpts = .{ + .freq_base = 500_000, + .scaling = .{ .llama3 = .{ + .factor = 32, + .high_freq_factor = 4, + .low_freq_factor = 1, + .original_max_position_embeddings = 8192, + } }, + }; + const llama_freq = [_]f32{ 1.000000000000e+00, 6.636012792587e-01, 4.403666257858e-01, 2.922278344631e-01, 1.939227581024e-01, 1.286873817444e-01, 8.539710193872e-02, 5.666961893439e-02, 3.760603070259e-02, 2.495540864766e-02, 1.656044088304e-02, 1.098952908069e-02, 7.292665075511e-03, 4.839421249926e-03, 3.211446106434e-03, 1.290548010729e-03, 4.295567050576e-04, 9.708286233945e-05, 1.946163865796e-05, 1.291476746701e-05, 8.570255886298e-06, 5.687232260243e-06, 3.774054448513e-06, 2.504467147446e-06, 1.661967417022e-06, 1.102883629756e-06, 7.318749339902e-07, 4.856731266045e-07, 3.222932889457e-07, 2.138742303259e-07, 1.419272024350e-07, 9.418306490261e-08 }; + + var inv_freq: @TypeOf(llama_freq) = undefined; + _invFreq(llama_conf, &inv_freq); + for (llama_freq, inv_freq, 0..) |expected, actual, i| { + errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, llama_freq, inv_freq }); + try std.testing.expectApproxEqRel(expected, actual, 1e-5); + } } test "real/img" { const platform = zml.testing.env(); const Fns = struct { - fn testSplitMergeIsId(impl: RopeOpts.Implementation) Tensor { + fn testSplitMergeIsId(layout: RopeOpts.Layout) Tensor { const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, impl); - const y = mergeRealImg(real, imag, impl); - const real2, const imag2 = splitRealImg(y, impl); + const real, const imag = splitRealImg(x, layout); + const y = mergeRealImg(real, imag, layout); + const real2, const imag2 = splitRealImg(y, layout); return real.cmp(.EQ, real2).flatten(0).convert(.i32).sum(-1).add( imag.cmp(.EQ, imag2).flatten(0).convert(.i32).sum(-1), ); @@ -312,13 +415,13 @@ test rope { { // Convert input to the requested format const real, const imag = splitRealImg(input, .sequential); - input = mergeRealImg(real, imag, opts.impl); + input = mergeRealImg(real, imag, opts.layout); } var res = rope(input, null, opts).squeeze(0); { // Convert back to sequential - const real, const imag = splitRealImg(res, opts.impl); + const real, const imag = splitRealImg(res, opts.layout); res = mergeRealImg(real, imag, .sequential); } return res; @@ -328,8 +431,8 @@ test rope { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. const x = try zml.Buffer.fromSlice(platform, .{ .b = 1, .s = 5, .hd = 4 }, &[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5); - const res1 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .impl = .interleaved } }); - const res2 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .impl = .sequential } }); + const res1 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }); + const res2 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }); try zml.testing.expectClose(res1, res2, 1e-4); } From a53303e9e48bc8136d1fefbbcd494f15dd85aa8a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 30 May 2025 08:28:51 +0000 Subject: [PATCH 21/22] runtimes/cuda: bump PJRT plugin --- runtimes/cuda/cuda.bzl | 4 ++-- runtimes/cuda/libpjrt_cuda.BUILD.bazel | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index 82d1e448..0c432802 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -202,8 +202,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "64029cd3d68118b166198e3246877ed706ed35eb732b1770a9bf530b5b0a8ab4", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v9.0.0/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "0cd5b3214e32d39f501e02d38d5166ffa7eeb9d4852cf2108c3f4f74df3fe061", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index 17318a49..8881a016 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -25,8 +25,7 @@ copy_to_directory( cc_import( name = "libpjrt_cuda", data = [":sandbox"], - shared_library = "@zml//runtimes/cuda:libpjrt_cuda.so", - #shared_library = "libpjrt_cuda.so", + shared_library = "libpjrt_cuda.so", add_needed = ["libzmlxcuda.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", From d53adb15286f85ef271464035727875f12c67ecb Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 2 Jun 2025 13:19:08 +0000 Subject: [PATCH 22/22] pjrt: added CompiledMemoryStats --- pjrt/pjrt.zig | 37 +++++++++++++++++++++++++++++++++++++ zml/pjrtx.zig | 3 ++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 8ece2eef..437e908c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -592,6 +592,43 @@ pub const Executable = opaque { .deleter = @ptrCast(ret.serialized_executable_deleter.?), }; } + + pub fn getCompiledMemoryStats(self: *const Executable, api: *const Api) ApiError!CompiledMemoryStats { + const ret = try api.call(.PJRT_Executable_GetCompiledMemoryStats, .{ + .executable = self.inner(), + }); + + return .{ + .generated_code_size_in_bytes = ret.generated_code_size_in_bytes, + .argument_size_in_bytes = ret.argument_size_in_bytes, + .output_size_in_bytes = ret.output_size_in_bytes, + .alias_size_in_bytes = ret.alias_size_in_bytes, + .temp_size_in_bytes = ret.temp_size_in_bytes, + .host_generated_code_size_in_bytes = ret.host_generated_code_size_in_bytes, + .host_argument_size_in_bytes = ret.host_argument_size_in_bytes, + .host_output_size_in_bytes = ret.host_output_size_in_bytes, + .host_alias_size_in_bytes = ret.host_alias_size_in_bytes, + .host_temp_size_in_bytes = ret.host_temp_size_in_bytes, + }; + } +}; + +pub const CompiledMemoryStats = struct { + // Mirrors xla::CompiledMemoryStats. + // Device default memory (e.g., HBM for GPU/TPU) usage stats. + generated_code_size_in_bytes: i64, + argument_size_in_bytes: i64, + output_size_in_bytes: i64, + // much: How argument is reused for output. + alias_size_in_bytes: i64, + temp_size_in_bytes: i64, + + // memory: Host usage stats. + host_generated_code_size_in_bytes: i64, + host_argument_size_in_bytes: i64, + host_output_size_in_bytes: i64, + host_alias_size_in_bytes: i64, + host_temp_size_in_bytes: i64, }; pub const LoadedExecutable = opaque { diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 419b8542..90e6d693 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -19,6 +19,7 @@ pub const Error = pjrt.Error; pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; pub const SerializeResult = pjrt.SerializeResult; pub const Executable = pjrt.Executable; +pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; pub const ExecuteError = ApiError; pub const Memory = pjrt.Memory; @@ -60,7 +61,7 @@ pub const Client = opaque { pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); - return .{ @ptrCast(buffer), @ptrCast(event_)}; + return .{ @ptrCast(buffer), @ptrCast(event_) }; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable {