diff --git a/MODULE.bazel b/MODULE.bazel index c664ef19..020e42dc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -42,7 +42,6 @@ protoc.toolchain( version = "v29.2", ) use_repo(protoc, "com_google_protobuf", "toolchains_protoc_hub") - register_toolchains("@toolchains_protoc_hub//:all") zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") @@ -52,14 +51,11 @@ zig.mirrors(urls = [ "https://mirror.zml.ai/zig", ]) use_repo(zig, "zig_toolchains") - register_toolchains("@rules_zig//zig/target:all") - register_toolchains("@zig_toolchains//:all") toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") use_repo(toolchains, "zig_sdk") - register_toolchains( "@zig_sdk//toolchain:linux_amd64_gnu.2.31", "@zig_sdk//toolchain:linux_arm64_gnu.2.31", @@ -102,7 +98,6 @@ use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib") zls = use_extension("//third_party/zls:zls.bzl", "repo") use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux") - register_toolchains("//third_party/zls:all") llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") diff --git a/async/async.zig b/async/async.zig index 79026da2..1ce5148c 100644 --- a/async/async.zig +++ b/async/async.zig @@ -551,12 +551,9 @@ pub fn logFn(comptime fallbackLogFn: LogFn) LogFn { var bw = std.io.bufferedWriter(stderr); const writer = bw.writer(); - var mutex = Self.mu orelse blk: { - Self.mu = Mutex.init(); - break :blk Self.mu.?; - }; - mutex.lock(); - defer mutex.unlock(); + Self.mu = Self.mu orelse Mutex.init(); + Self.mu.?.lock(); + defer Self.mu.?.unlock(); nosuspend { writer.print(level_txt ++ prefix2 ++ format ++ "\n", args) catch return; bw.flush() catch return; diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index d7d8673c..ab393473 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -735,9 +735,12 @@ pub fn convolution( pub const CustomCallOpts = struct { call_target_name: [:0]const u8, has_side_effect: bool, - backend_config: [:0]const u8 = &.{}, + backend_config: union(enum) { + str: [:0]const u8, + dict: mlir.DictionaryAttribute, + }, api_version: i32, - output_operand_aliases: []const i64, + output_operand_aliases: []const i64 = &.{}, }; pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { @@ -757,13 +760,91 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, opts.api_version).as(mlir.Attribute).? }, .{ "call_target_name", mlir.StringAttribute.init(ctx, opts.call_target_name).as(mlir.Attribute).? }, .{ "has_side_effect", mlir.BoolAttribute.init(ctx, opts.has_side_effect).as(mlir.Attribute).? }, - .{ "backend_config", mlir.StringAttribute.init(ctx, opts.backend_config).as(mlir.Attribute).? }, + .{ + "backend_config", switch (opts.backend_config) { + .str => |v| mlir.StringAttribute.init(ctx, v).as(mlir.Attribute).?, + .dict => |v| v.as(mlir.Attribute).?, + }, + }, .{ "output_operand_aliases", mlir.ArrayAttribute.init(ctx, output_operand_aliases).as(mlir.Attribute).? }, }, .location = location, }); } +pub fn custom_call_alloc(allocator: std.mem.Allocator, ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + + var arena = arena_state.allocator(); + + const output_operand_aliases = arena.alloc(mlir.Attribute, opts.output_operand_aliases.len) catch unreachable; + for (opts.output_operand_aliases, 0..) |alias, i| { + output_operand_aliases[i] = OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).as(mlir.Attribute).?; + } + + const indices_of_shape_operands = b: { + var indices = std.ArrayList(i64).initCapacity(arena, res_types.len) catch unreachable; + for (0..res_types.len) |indice| { + indices.appendAssumeCapacity(@intCast(indice)); + } + const indices_of_shape_type = mlir.RankedTensorType.init(&.{@intCast(res_types.len)}, mlir.IntegerType(.i32).init(ctx).as(mlir.Type).?).as(mlir.Type).?; + break :b mlir.DenseIntOrFPElementsAttribute(.i32).init(indices_of_shape_type, std.mem.sliceAsBytes(indices.items)); + }; + _ = indices_of_shape_operands; + + const operand_layouts = b: { + var layouts = std.ArrayList(mlir.Attribute).initCapacity(arena, inputs.len) catch unreachable; + const MINOR_TO_MAJOR = [8]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; + for (inputs) |input| { + const len = MINOR_TO_MAJOR.len; + const input_type = input.getType().as(mlir.ShapedType).?; + const layout = MINOR_TO_MAJOR[len - input_type.rank() ..]; + const layout_shape = mlir.RankedTensorType.init(&.{@intCast(layout.len)}, mlir.IndexType.init(ctx).as(mlir.Type).?).as(mlir.Type).?; + const layout_attribute = mlir.DenseIntOrFPElementsAttribute(.i64).fromRaw(layout_shape, std.mem.sliceAsBytes(layout)).as(mlir.Attribute).?; + layouts.appendAssumeCapacity(layout_attribute); + } + + const operand_layouts = mlir.ArrayAttribute.init(ctx, layouts.items).as(mlir.Attribute).?; + break :b operand_layouts; + }; + const result_layouts = b: { + var layouts = std.ArrayList(mlir.Attribute).initCapacity(arena, res_types.len) catch unreachable; + const MINOR_TO_MAJOR = [8]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; + for (res_types) |res_raw_type| { + const len = MINOR_TO_MAJOR.len; + const res_type = res_raw_type.as(mlir.ShapedType).?; + const layout = MINOR_TO_MAJOR[len - res_type.rank() ..]; + const layout_shape = mlir.RankedTensorType.init(&.{@intCast(layout.len)}, mlir.IndexType.init(ctx).as(mlir.Type).?).as(mlir.Type).?; + const layout_attribute = mlir.DenseIntOrFPElementsAttribute(.i64).fromRaw(layout_shape, std.mem.sliceAsBytes(layout)).as(mlir.Attribute).?; + layouts.appendAssumeCapacity(layout_attribute); + } + + const result_layouts = mlir.ArrayAttribute.init(ctx, layouts.items).as(mlir.Attribute).?; + break :b result_layouts; + }; + + return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ + .operands = inputs, + .results = res_types, + .attributes = &.{ + .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, opts.api_version).as(mlir.Attribute).? }, + .{ "call_target_name", mlir.StringAttribute.init(ctx, opts.call_target_name).as(mlir.Attribute).? }, + .{ "has_side_effect", mlir.BoolAttribute.init(ctx, opts.has_side_effect).as(mlir.Attribute).? }, + .{ + "backend_config", switch (opts.backend_config) { + .str => |v| mlir.StringAttribute.init(ctx, v).as(mlir.Attribute).?, + .dict => |v| v.as(mlir.Attribute).?, + }, + }, + .{ "output_operand_aliases", mlir.ArrayAttribute.init(ctx, output_operand_aliases).as(mlir.Attribute).? }, + .{ "operand_layouts", operand_layouts }, + .{ "result_layouts", result_layouts }, + }, + .location = location, + }); +} + pub fn sharding(ctx: mlir.Context, inputs: []const mlir.Value, sharding_spec: mlir.StringAttribute, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ .operands = inputs, diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 2dc2b9f3..bbe9d3ee 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -649,6 +649,10 @@ pub fn DenseIntOrFPElementsAttribute(comptime dt: DenseElementsAttributeTypes) t return Attr.wrap(initFn(shaped_type.inner(), @intCast(values.len), @ptrCast(@alignCast(values.ptr)))); } + pub fn fromRaw(shaped_type: Type, raw: []const u8) Attr { + return Attr.wrap(c.mlirDenseElementsAttrRawBufferGet(shaped_type.inner(), raw.len, raw.ptr)); + } + pub fn get(self: Attr, pos: usize) ZigOutDataType { return getValue(self.inner(), @intCast(pos)); } @@ -1239,6 +1243,21 @@ pub const Type = struct { } }; +pub const IndexType = struct { + _inner: c.MlirType, + + pub usingnamespace MlirHelpers(IndexType, .{ + .is_null_fn = c.mlirTypeIsNull, + .dump_fn = c.mlirTypeDump, + .equal_fn = c.mlirTypeEqual, + .print_fn = c.mlirTypePrint, + }); + + pub fn init(ctx: Context) IndexType { + return IndexType.wrap(c.mlirIndexTypeGet(ctx.inner())); + } +}; + pub const IntegerTypes = enum { i1, i4, diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index f48b6361..e80b97db 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -502,6 +502,16 @@ pub const Device = opaque { }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) []*const Memory { + const ret = api.call(.PJRT_Device_AddressableMemories, .{ + .device = self.inner(), + }) catch unreachable; + if (ret.memories) |memories| { + return @ptrCast(memories[0..ret.num_memories]); + } + return &.{}; + } }; pub const DeviceDescription = opaque { @@ -811,7 +821,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, diff --git a/zml/aio.zig b/zml/aio.zig index 27abbd8a..9740e891 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -599,6 +599,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; obj.* = try zml.Buffer.from(platform, buf_with_metadata); + _ = try obj.awaitt(); } else { return error.BufferNotFound; }; diff --git a/zml/buffer.zig b/zml/buffer.zig index 20a6fcb7..16bf59cd 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -55,7 +55,7 @@ pub const Buffer = struct { _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; - pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const Shards = std.BoundedArray(Shard, MAX_NUM_SHARDS); /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { @@ -79,44 +79,49 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - .{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buf.data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + }); + res._shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + .ready_event = event, }); - - frames.appendAssumeCapacity(frame); } - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); - res._shards.appendAssumeCapacity(pjrt_buffer); - } return res; } + pub fn awaitt(self: *Buffer) !*Buffer { + for (self._shards.slice()) |*shard| { + try shard.awaitt(); + } + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); + for (pjrt_buffers) |pjrt_buffer| { + shards.appendAssumeCapacity(.{ + .api = platform.pjrt_api, + .buffer = pjrt_buffer, + }); + } return .{ ._api = platform.pjrt_api, ._shape = shape_, @@ -197,7 +202,9 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, .data = &bytes }; - return try from(platform, host_buffer); + var buf = try from(platform, host_buffer); + _ = try buf.awaitt(); + return buf; } test constant { @@ -262,9 +269,9 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.awaitt(self._api); } return res; } @@ -274,22 +281,23 @@ pub const Buffer = struct { /// The returned `HostBuffer` doesn't own the memory. pub fn toHost(self: Buffer, output: []u8) !HostBuffer { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, output); + var hb = HostBuffer.fromBytes(self.shape(), output); + hb._event = maybe_event; + hb._api = self._api; + hb._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return hb; } /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); + var output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); - if (maybe_event) |event| { - try event.await_(self._api); - } + const maybe_event = try self._shards.get(0).buffer.toHostBuffer(self._api, @constCast(output.data)); + output._event = maybe_event; + output._api = self._api; + output._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; return output; } @@ -297,8 +305,8 @@ pub const Buffer = struct { /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *const Buffer) void { - for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._api); + for (self._shards.constSlice()) |shard| { + shard.buffer.deinit(self._api); } } diff --git a/zml/exe.zig b/zml/exe.zig index 48329b9b..f398744c 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -211,11 +211,11 @@ pub const BaseExe = struct { .non_donatable_input_indices = &.{}, }) catch unreachable; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn serialize(self: BaseExe, writer: anytype) !void { @@ -317,7 +317,7 @@ fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { const model_sharding = ctx.buffers.len; stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {}-sharded tensor into a {}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); for (buffer._shards.constSlice(), 0..) |shard, d| { - ctx.buffers[d][ctx.index] = shard; + ctx.buffers[d][ctx.index] = shard.buffer; } ctx.index += 1; } @@ -345,7 +345,7 @@ fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Bu ctx.index += 1; if (i >= ctx.buffer_shapes.len) return; - var shards: Buffer.Shards = .{}; + var shards: std.BoundedArray(*pjrt.Buffer, Buffer.MAX_NUM_SHARDS) = .{}; for (ctx.buffers) |buff| { shards.appendAssumeCapacity(buff[i]); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index cd7e3a60..da96cff9 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -1,6 +1,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); + const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -22,6 +24,9 @@ pub const HostBuffer = struct { managed: u5, unmanaged, } = .unmanaged, + _ready: bool = true, + _api: ?*const pjrt.Api = null, + _event: ?*pjrt.Event = null, /// Allocates a HostBuffer with the given shape. /// The memory is left undefined. @@ -255,6 +260,21 @@ pub const HostBuffer = struct { }; } + pub fn awaitt(self: *HostBuffer) !*HostBuffer { + if (self._ready) { + return self; + } + if (self._event) |ev| { + if (self._api) |api| { + try ev.awaitt(api); + } else { + stdx.debug.panic("Got an event but no API. This should not happen.", .{}); + } + } + self._ready = true; + return self; + } + pub fn format( self: HostBuffer, comptime fmt: []const u8, diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index 854da729..c8a49461 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -126,7 +126,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { &.{ q.value(), k.value(), v.value(), bias.value() }, .{ .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", - .backend_config = backend_config, + .backend_config = .{ .str = backend_config }, .api_version = 2, .has_side_effect = false, .output_operand_aliases = &.{}, diff --git a/zml/ops.zig b/zml/ops.zig index c73dc689..643ab9f2 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1160,3 +1160,45 @@ inline fn toI64(values: anytype) []i64 { for (values, 0..) |val, i| res[i] = @intCast(val); return res[0..values.len]; } + +pub const TritonOps = struct { + debug: bool = false, + name: [:0]const u8, + ir: [:0]const u8, + grid: [3]i32, + num_stages: i32, + num_warps: i32, +}; + +/// Generate an MLIR call to the given member function with the given tensors. +pub fn triton(args: anytype, res_shape: Shape, opts: TritonOps) Tensor { + const ctx = CompilationContext.current(); + + var values: [args.len]mlir.Value = undefined; + ctx.extractValues(&args, &values); + + const attrs = mlir.DictionaryAttribute.init(ctx.mlirCtx(), &.{ + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "name"), mlir.StringAttribute.init(ctx.mlirCtx(), opts.name).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "ir"), mlir.StringAttribute.init(ctx.mlirCtx(), opts.ir).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "grid_x"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.grid[0])).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "grid_y"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.grid[1])).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "grid_z"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.grid[2])).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "num_stages"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.num_stages)).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "num_warps"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.num_warps)).as(mlir.Attribute).?), + }); + + const op = dialect.stablehlo.custom_call_alloc( + ctx.allocator(), + ctx.mlirCtx(), + &values, + .{ + .call_target_name = "__gpu$xla.gpu.triton", + .backend_config = .{ .dict = attrs }, + .api_version = 4, + .has_side_effect = false, + }, + &.{mlir.ext.mlirType(ctx.mlirCtx(), res_shape)}, + ctx.mlirCtx().location(@src()), + ); + return Tensor._result(res_shape, op.result(0)); +} diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index a642979c..ea59219a 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -28,8 +28,8 @@ pub const Error = pjrt.Error; pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; pub const SerializeResult = pjrt.SerializeResult; pub const Executable = pjrt.Executable; -pub const ExecuteError = ApiError; pub const Memory = pjrt.Memory; +pub const ExecuteError = ApiError; fn InnerMixin(comptime innerT: type) type { return struct { @@ -63,13 +63,9 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return @ptrCast(buffer); + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -211,7 +207,8 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) ApiError!void { + pub const await_ = awaitt; + pub fn awaitt(self: *Event, api: *const Api) !void { defer self.deinit(api); if (self.isReady(api)) { @@ -267,13 +264,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable {