From fe7206628bcb42aea485f74f77740ac53074b4df Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 31 Mar 2025 15:16:17 +0000 Subject: [PATCH 01/15] zml/buffer: resuscitate async buffers --- zml/aio.zig | 2 +- zml/buffer.zig | 77 +++++++++++++++++++++++++++++++++++--------------- zml/pjrtx.zig | 14 ++++----- 3 files changed, 61 insertions(+), 32 deletions(-) diff --git a/zml/aio.zig b/zml/aio.zig index 98299908..38f996f9 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -654,7 +654,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from(platform, buf_with_metadata); + obj.* = try zml.Buffer.from(platform, buf_with_metadata, .{}); } else { log.err("Buffer not found: {s}", .{prefix}); diff --git a/zml/buffer.zig b/zml/buffer.zig index e0829d39..624c2957 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -68,8 +68,12 @@ pub const Buffer = struct { pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const FromOptions = struct { + wait: bool = true, + }; + /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { + pub fn from(platform: Platform, host_buffer: HostBuffer, opts: FromOptions) !Buffer { var res: Buffer = .{ ._api = platform.pjrt_api, ._shape = host_buffer.shape(), @@ -90,7 +94,6 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { // If no sharding if found, the given buffer is replicated on all devices. @@ -99,29 +102,39 @@ pub const Buffer = struct { break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - pjrt.Client.BufferFromHostBufferArgs{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, pjrt.Client.BufferFromHostBufferArgs{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, }); - frames.appendAssumeCapacity(frame); - } + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); res._shards.appendAssumeCapacity(pjrt_buffer); } + + if (opts.wait) { + res = try res.awaitt(); + } + return res; } + pub fn awaitt(self: Buffer) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.await_(self._api); + } + } + + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -139,20 +152,40 @@ pub const Buffer = struct { /// return a Buffer with the given dimensions. pub fn fromSlice(platform: Platform, dimz: anytype, s: anytype) !Buffer { const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); - return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); + return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s)), .{}); } /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromBytes(platform: Platform, sh: Shape, data: []const u8) !Buffer { - return from(platform, HostBuffer.fromBytes(sh, data)); + return from(platform, HostBuffer.fromBytes(sh, data), .{}); } /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { const host_buffer = HostBuffer.fromArray(&arr); - return try from(platform, host_buffer); + return try from(platform, host_buffer, .{}); + } + + /// Copies the given Zig slice to the accelerator memory and + /// return a Buffer with the given dimensions. + pub fn fromSliceOpts(platform: Platform, dimz: anytype, s: anytype, opts: FromOptions) !Buffer { + const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); + return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s)), opts); + } + + /// Copies the given Zig slice to the accelerator memory and + /// return a Buffer with the given dimensions. + pub fn fromBytesOpts(platform: Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { + return from(platform, HostBuffer.fromBytes(sh, data), opts); + } + + /// Copies the given Zig array to the accelerator memory and + /// return a Buffer using the array shape. + pub fn fromArrayOpts(platform: Platform, arr: anytype, opts: FromOptions) !Buffer { + const host_buffer = HostBuffer.fromArray(&arr); + return try from(platform, host_buffer, opts); } /// Creates a Buffer with a single element. @@ -185,7 +218,7 @@ pub const Buffer = struct { ._strides = [1]i64{0} ** Shape.MAX_RANK, .data = x.constSlice(), }; - return try from(platform, host_buffer); + return try from(platform, host_buffer, .{}); } // To speed up copies, duplicate the scalar value into a vector, @@ -208,7 +241,7 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, .data = &bytes }; - return try from(platform, host_buffer); + return try from(platform, host_buffer, .{}); } test constant { diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index bd094ead..de1bf570 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -57,13 +57,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return @ptrCast(buffer); + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_)}; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -265,13 +261,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 2734184b3d79585fdc85ba593a6403d90e70e516 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 31 Mar 2025 15:16:52 +0000 Subject: [PATCH 02/15] zml/aio: implement awaitAll --- zml/aio.zig | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zml/aio.zig b/zml/aio.zig index 38f996f9..cfcbcd27 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -632,8 +632,11 @@ fn findSimilarBufferKeys(original_key: []const u8, store: BufferStore, temp_allo /// deinit all buffers in the given struct pub fn awaitAll(buffers: anytype) !void { - // TODO: implement once we have async buffers. - _ = buffers; + zml.meta.visit((struct { + fn cb(_: void, buffer: *zml.Buffer) void { + buffer.* = buffer.awaitt() catch unreachable; + } + }).cb, {}, buffers); } fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, buffer_store: BufferStore, obj: anytype, platform: zml.Platform) !void { From 4d226f4752ac0faa848d4931c8dcd9da04700a40 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 31 Mar 2025 15:17:06 +0000 Subject: [PATCH 03/15] zml/exe: disable execute awaiting --- zml/exe.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index 946410c6..c59f8e4a 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -211,11 +211,11 @@ pub const BaseExe = struct { .non_donatable_input_indices = &.{}, }) catch unreachable; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + //for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + //} } pub fn serialize(self: BaseExe, writer: anytype) !void { From 2a0fdd3c1901c4f443add1aa1eef70b151f0cee2 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 10 Apr 2025 08:58:43 +0000 Subject: [PATCH 04/15] mlir/dialects/stablehlo: added defaults for (operand|result)_layouts --- mlir/dialects/stablehlo.zig | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index 63283219..24466ddf 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -786,12 +786,32 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); } + const MINOR_TO_MAJOR = blk: { + const MAX_RANK = 8; + var ret: [MAX_RANK]usize = undefined; + for (0..MAX_RANK) |i| { + ret[i] = @intCast(MAX_RANK - i - 1); + } + break :blk ret; + }; + if (opts.operand_layouts) |layouts| { var operand_layouts: std.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; for (layouts) |ol| { operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); } attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); + } else { + const operand_layouts = blk: { + var ret: std.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; + for (inputs) |input| { + const ranked_type = input.getType().as(mlir.RankedTensorType); + const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; + ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); + } + break :blk ret; + }; + attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); } if (opts.result_layouts) |layouts| { @@ -800,6 +820,17 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); } attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); + } else { + const result_layouts = blk: { + var ret: std.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; + for (res_types) |t| { + const ranked_t = t.as(mlir.RankedTensorType); + const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; + ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); + } + break :blk ret; + }; + attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); } attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); From 62f664ac56eea2012293f108b2df18cb7ffb4d3e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 10 Apr 2025 08:59:34 +0000 Subject: [PATCH 05/15] zml/nn: bind paged sdpa --- zml/nn.zig | 2 +- zml/nn/cuda.zig | 151 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index c3120d26..b5560f09 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,7 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const cuda = @import("nn/cuda.zig"); +pub const cuda = @import("nn/cuda.zig"); const helpers = @import("helpers.zig"); const meta = @import("meta.zig"); const ops = @import("ops.zig"); diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index 8652f9bc..e087bc61 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -138,3 +138,154 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const result = Tensor._result(q.shape(), op.result(0)).transpose(q_.shape()); return result; } + +pub const SdpaPagedOpts = struct { + attn_mask: ?Tensor = null, + scale: ?Tensor = null, + max_seq_len_kv: usize, + max_input_len: usize, +}; + +pub fn layout_constraint(input: Tensor, output_shape: Shape) Tensor { + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + &.{input.value()}, + .{ + .call_target_name = "LayoutConstraint", + .api_version = 2, + .has_side_effect = false, + .output_operand_aliases = &.{}, + }, + &.{ + mlir.ext.mlirType(mlir_ctx, output_shape), + //mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type).?).asType(), + }, + loc, + ); + const result = Tensor._result(output_shape, op.result(0)); + return result; +} + +pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, sequence_length_kv: Tensor, page_table: Tensor, seq_offset: Tensor, opts: SdpaPagedOpts) Tensor { + const ctx = CompilationContext.current(); + //var q = q_; + //var k = k_; + //var v = v_; + //const q = layout_constraint(q_, q_.shape().transpose(.{ .b, .h, .q, .hd })); + //var k = layout_constraint(k_, k_.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + //const v = layout_constraint(v_, v_.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + const q = q_.transpose(.{ .b, .h, .q, .hd }); + var k = k_.transpose(.{ .page, .h, .k_chunk, .hd }); + const v = v_.transpose(.{ .page, .h, .k_chunk, .hd }); + + const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); + const kernel_scale = if (opts.scale) |_| 1.0 else sqrtHeadDim; + if (opts.scale) |s| { + k = k.mul(s); + } + + var buffer: [4096]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buffer); + const allocator = fba.allocator(); + + const backend_config = + std.fmt.allocPrintZ( + allocator, + \\{{ + \\ "operation_queue_id":"0", + \\ "wait_on_operation_queues":[], + \\ "cudnn_fmha_backend_config": {{ + \\ "algorithm": {{ + \\ "algo_id": "0", + \\ "math_type": "TENSOR_OP_MATH", + \\ "tuning_knobs": {{"17": "1", "24": "0"}}, + \\ "is_cudnn_frontend": true, + \\ "workspace_size": "0", + \\ }}, + \\ "fmha_scale": {}, + \\ "dropout_rate": 0, + \\ "intermediate_tensor_shape": {{ + \\ "element_type": "{s}", + \\ "dimensions": ["{}", "{}", "{}", "{}"], + \\ "tuple_shapes": [], + \\ "layout": {{ + \\ "dim_level_types": [], + \\ "dim_unique": [], + \\ "dim_ordered": [], + \\ "minor_to_major": ["3", "2", "1", "0"], + \\ "tiles": [], + \\ "element_size_in_bits": "0", + \\ "memory_space": "0", + \\ "index_primitive_type": "PRIMITIVE_TYPE_INVALID", + \\ "pointer_primitive_type": "PRIMITIVE_TYPE_INVALID", + \\ "dynamic_shape_metadata_prefix_bytes": "0", + \\ }}, + \\ "is_dynamic_dimension": [false, false, false, false], + \\ }}, + \\ "seed": 0, + \\ "is_flash_attention": true, + \\ "mask_type": "NO_MASK", + \\ "bmm1_dot_dimension_numbers": {{ + \\ "lhs_contracting_dimensions": ["3"], + \\ "rhs_contracting_dimensions": ["3"], + \\ "lhs_batch_dimensions": ["0", "1"], + \\ "rhs_batch_dimensions": ["0", "1"], + \\ }}, + \\ "bmm2_dot_dimension_numbers": {{ + \\ "lhs_contracting_dimensions": ["3"], + \\ "rhs_contracting_dimensions": ["2"], + \\ "lhs_batch_dimensions": ["0", "1"], + \\ "rhs_batch_dimensions": ["0", "1"], + \\ }}, + \\ "max_sequence_length_kv": {}, + \\ "max_input_len": {}, + \\ }} + \\}} + , + .{ + kernel_scale, + elementTypeFromDataType(q.dtype()), + q.dim(.b), + q.dim(.h), + q.dim(.q), + k.dim(.k_chunk), + opts.max_seq_len_kv, + opts.max_input_len, + }, + ) catch unreachable; + + var bias = Tensor.constant(Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .k = opts.max_seq_len_kv }, q.dtype()), Data.init(q.dtype(), 0)); + + if (opts.attn_mask) |attn_mask| { + bias = bias.add(attn_mask.broad(bias.shape())); + } + + //q = q_.transpose(.{ .b, .h, .q, .hd }); + //q = layout_constraint(q, q.shape()); + //k = layout_constraint(k, k.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + //v = layout_constraint(v, v.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + const output_shape = Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .hd = q.dim(.hd) }, q.dtype()); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + &.{ q.value(), k.value(), v.value(), sequence_length_q.value(), sequence_length_kv.value(), page_table.value(), page_table.value(), seq_offset.value() }, + .{ + .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", + .backend_config = mlir.Attribute.string(mlir_ctx, backend_config), + .api_version = .status_returning, + .has_side_effect = false, + .output_operand_aliases = &.{}, + }, + &.{ + mlir.ext.mlirType(mlir_ctx, output_shape), + mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type)).as(mlir.Type), + }, + loc, + ); + const result = Tensor._result(output_shape, op.result(0)).transpose(q_.shape()); + return result; +} From 2d86823127d13369479567fa15f14105cd1f1a3e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 10 Apr 2025 19:37:20 +0000 Subject: [PATCH 06/15] wip --- pjrt/pjrt.zig | 4 ++-- zml/buffer.zig | 10 ++++++++++ zml/ops.zig | 4 ++-- zml/pjrtx.zig | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9dbd5aac..3ded8524 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,7 +31,7 @@ fn pjrtStructSize(comptime T: type) usize { return @field(c, typedef_name ++ "_STRUCT_SIZE"); } -inline fn pjrtStruct(v: anytype) @TypeOf(v) { +pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { var ret = v; ret.struct_size = pjrtStructSize(@TypeOf(v)); return ret; @@ -925,7 +925,7 @@ pub const Memory = opaque { pub fn kind(self: *const Memory, api: *const Api) Kind { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable[0..ret.kind_size]; + const kind_ = ret.kind[0..ret.kind_size]; return std.meta.stringToEnum(Kind, kind_) orelse unreachable; } diff --git a/zml/buffer.zig b/zml/buffer.zig index 624c2957..c6277fa9 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -394,6 +394,16 @@ pub const Buffer = struct { if (self._shards.len == 1) return false; return @reduce(.Or, self._shape._sharding_info); } + + pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { + var new_shards: Buffer.Shards = .{}; + for (self._shards.slice()) |shard| { + const new_shard = try shard.copyToMemory(self._api, memory); + new_shards.appendAssumeCapacity(new_shard); + } + + return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; + } }; pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { diff --git a/zml/ops.zig b/zml/ops.zig index 1765888a..efe5d36c 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -22,8 +22,8 @@ const ShapeOf = @import("tensor.zig").ShapeOf; const Tensor = @import("tensor.zig").Tensor; const EnumLiteral = @TypeOf(.enum_literal); -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; +pub const dialect = struct { + pub const stablehlo = @import("mlir/dialects").stablehlo; }; const log = std.log.scoped(.@"zml/tensor"); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index de1bf570..b2b04c1d 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -116,7 +116,7 @@ pub const Client = opaque { return self.inner().addressableMemories(api); } - pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*Memory { + pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { for (self.addressableMemories(api)) |mem| { if (mem.kind(api) == kind) { return mem; @@ -178,7 +178,7 @@ pub const Buffer = opaque { } pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(self.inner().copyToMemory(api, memory_)); + return @ptrCast(try self.inner().copyToMemory(api, memory_)); } pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { From ddd170578dd5744c6e253a8209d48762d77fcc10 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 18 Apr 2025 14:29:19 +0000 Subject: [PATCH 07/15] zml/tensor.zig: fix returned value in Tensor.toMemory --- zml/tensor.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index b248ea20..79c2f2c9 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -206,8 +206,6 @@ pub const Tensor = struct { const ctx = self.getContext(); const mlir_ctx = ctx.mlirCtx(); if (ctx.target() == .cpu) return self; - var res = self; - res._output_memory_kind = kind; const memory_kind = @tagName(kind.toPjrtMemory()); @@ -223,7 +221,9 @@ pub const Tensor = struct { .api_version = .original, }, &.{self.value().getType()}, mlir_ctx.location(@src())); - return _result(res._shape, op.result(0)); + var res = _result(self._shape, op.result(0)); + res._output_memory_kind = kind; + return res; }, .buffer_id => { var res = self; From 3eef25e8cb3bd5900dd5b79617b8007c5099d1fa Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 18 Apr 2025 14:30:14 +0000 Subject: [PATCH 08/15] pjrt: bind PJRT_Device_AddressableMemories --- pjrt/pjrt.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 3ded8524..f9a13e31 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -502,6 +502,13 @@ pub const Device = opaque { }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) ApiError![]const *Memory { + const ret = try api.call(.PJRT_Device_AddressableMemories, .{ + .device = self.inner(), + }); + return @ptrCast(ret.memories[0..ret.num_memories]); + } }; pub const DeviceDescription = opaque { From ee263eb55e189559c54afc2fad16fa5b67f8d377 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 23 Apr 2025 10:52:17 +0200 Subject: [PATCH 09/15] zml/buffer.zig: add a way to specify memory kind for buffer creation --- zml/buffer.zig | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index c6277fa9..0b86ae6a 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -70,6 +70,7 @@ pub const Buffer = struct { pub const FromOptions = struct { wait: bool = true, + memory: ?pjrt.Memory.Kind = null, }; /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -102,14 +103,25 @@ pub const Buffer = struct { break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, pjrt.Client.BufferFromHostBufferArgs{ + var args = pjrt.Client.BufferFromHostBufferArgs{ .data = buf.data, .buffer_type = buffer_type, .dims = buf.shape().dims(), .byte_strides = byte_strides, - .device = devices[i], .host_buffer_semantics = .ImmutableUntilTransferCompletes, - }); + }; + if (opts.memory) |memory_kind| { + const memories = try devices[i].addressableMemories(platform.pjrt_api); + const memory = for (memories) |m| { + const kind = m.kind(platform.pjrt_api); + if (kind == memory_kind) break m; + } else return error.NotFound; + args.memory = memory; + } else { + args.device = devices[i]; + } + + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); if (event) |ev| { ev.deinit(platform.pjrt_api); From ead9be07fb2edfc847574b7e39ab4dbb6b688ea5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 23 Apr 2025 10:52:55 +0200 Subject: [PATCH 10/15] runtimes/cuda: use local plugin --- runtimes/cuda/BUILD.bazel | 5 +++++ runtimes/cuda/libpjrt_cuda.BUILD.bazel | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index a7f8bf94..eebbcb12 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -34,6 +34,11 @@ zig_library( }), ) +exports_files( + ["libpjrt_cuda.so"], + visibility = ["//visibility:public"], +) + filegroup( name = "layers", srcs = [], diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index 8881a016..17318a49 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -25,7 +25,8 @@ copy_to_directory( cc_import( name = "libpjrt_cuda", data = [":sandbox"], - shared_library = "libpjrt_cuda.so", + shared_library = "@zml//runtimes/cuda:libpjrt_cuda.so", + #shared_library = "libpjrt_cuda.so", add_needed = ["libzmlxcuda.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", From 06cd5a7188d62b5bf9d041eee1da36e841724bc7 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 24 Apr 2025 17:29:59 +0000 Subject: [PATCH 11/15] zml/meta.zig: support union in map and mapAlloc --- zml/meta.zig | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/zml/meta.zig b/zml/meta.zig index 16680eb3..ead85d2d 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -59,6 +59,31 @@ pub fn MapType(From: type, To: type) type { .is_tuple = struct_infos.is_tuple, } }); }, + .@"union" => |union_info| { + const fields = union_info.fields; + var same: bool = true; + var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; + for (union_fields[0..], fields) |*union_field, field| { + const R = map(field.type); + if (R == field.type) { + union_field.* = field; + } else { + union_field.* = .{ + .name = field.name, + .type = R, + .alignment = @alignOf(R), + }; + same = false; + } + } + if (same) return T; + return @Type(.{ .@"union" = .{ + .layout = .auto, + .tag_type = union_info.tag_type, + .fields = union_fields[0..], + .decls = &.{}, + } }); + }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { .slice => if (ptr_info.is_const) @@ -144,6 +169,15 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam else => @field(to, field.name) = @field(from, field.name), } }, + .@"union" => |info| { + _ = info; // autofix + switch (from) { + inline else => |_, tag| { + to.* = @unionInit(ToStruct, @tagName(tag), undefined); + try mapAlloc(cb, allocator, ctx, @field(from, @tagName(tag)), &@field(to, @tagName(tag))); + }, + } + }, .array => for (from, to) |f, *t| { try mapAlloc(cb, allocator, ctx, f, t); }, @@ -174,7 +208,7 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } else { to.* = null; }, - .int, .float, .@"enum", .@"union" => to.* = from, + .int, .float, .@"enum" => to.* = from, else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), } } From bf7ce6034cfb6d3843605183df8dea0778cc6c32 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 28 Apr 2025 17:46:41 +0000 Subject: [PATCH 12/15] wip --- zml/pjrtx.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index b2b04c1d..089c985e 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const stdx = @import("stdx"); const asynk = @import("async"); const dialects = @import("mlir/dialects"); @@ -261,13 +262,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ + try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - }); + }}); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From de722633eb6d7896dca2fdc4b030189c22d97e17 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 29 Apr 2025 16:40:52 +0000 Subject: [PATCH 13/15] pjrt: wip wire command buffer compatibility --- pjrt/pjrt.zig | 11 +++++++++++ ...FFI-handler-registration-API-to-the-FFI-PjRt.patch | 9 +++++++-- zml/context.zig | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index f9a13e31..8ece2eef 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1164,17 +1164,27 @@ pub const NamedValue = extern struct { } }; +pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { + command_buffer_compatible = c.PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE, + _, +}; + // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub const CustomCallRegistry = extern struct { inner: *const c.PJRT_FFI_Register_Handler, + pub const RegisterFfiOptions = struct { + traits: RegisterHandlerTraits = @enumFromInt(0), + }; + pub fn registerFfi( self: *const CustomCallRegistry, api: *const Api, target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, + options: RegisterFfiOptions, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .api_version = 1, @@ -1183,6 +1193,7 @@ pub const CustomCallRegistry = extern struct { .handler = @ptrCast(@constCast(func)), .platform_name = platform_name.ptr, .platform_name_size = platform_name.len, + .traits = @intFromEnum(options.traits), }); const result = self.inner(&ret); if (result) |pjrt_c_error| { diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch index 58e3874d..bff748ec 100644 --- a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -31,10 +31,14 @@ diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_e index a33bd4aa9c..3309194538 100644 --- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h +++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); +@@ -66,12 +66,33 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); // Adds a user data to the execute context. typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); ++ typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ +struct PJRT_FFI_Register_Handler_Args { + size_t struct_size; + const char* target_name; @@ -43,8 +47,9 @@ index a33bd4aa9c..3309194538 100644 + void* handler; + const char* platform_name; + size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; +}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); + +// Registers an FFI call handler for a specific platform. +typedef PJRT_Error* PJRT_FFI_Register_Handler( diff --git a/zml/context.zig b/zml/context.zig index 03de62a3..874fea31 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -221,7 +221,7 @@ const CustomCall = struct { const registry = platform.pjrt_api.customCallRegistry(); if (registry) |reg| { - try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback); + try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback, .{}); } else { stdx.debug.panic("Registering custom calls failed", .{}); } From 0a3ca52705a809fda14087a0cdd1edaf39a6c5a6 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 29 Apr 2025 16:41:12 +0000 Subject: [PATCH 14/15] pjrtx: temporarily disable callBlocking for execute Now that we have command buffer, it's less of a problem --- zml/pjrtx.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 089c985e..f744622b 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -262,13 +262,20 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - }}); + }); + //try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + // .num_args = args.num_args, + // .arguments = @ptrCast(args.arguments), + // .results = @ptrCast(args.results), + // .events = @ptrCast(args.events), + // .non_donatable_input_indices = args.non_donatable_input_indices, + //}}); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 875b87feb601c551367b053a645eed7e875fbcfd Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 9 May 2025 16:28:20 +0000 Subject: [PATCH 15/15] zml/tensor.zig: Added `Tensor.zeroLike` --- zml/tensor.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zml/tensor.zig b/zml/tensor.zig index 79c2f2c9..8831985e 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1917,6 +1917,10 @@ pub const Tensor = struct { return _result(val.shape(), constant_op.result(0)); } + pub fn zeroLike(other: Tensor) Tensor { + return Tensor.constant(other.shape(), other.dtype().zero()); + } + /// Returns a Tensor containing the result of the outer product between the input Tensors. pub fn outer(self: Tensor, other: Tensor) Tensor { if (self.rank() + other.rank() == 1) {