diff --git a/MODULE.bazel b/MODULE.bazel index 10f43698..9dd9e760 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -98,7 +98,7 @@ llvm.configure( use_repo(llvm, "llvm-project") bazel_dep(name = "stablehlo", version = "20241021.0-1c0b606") -bazel_dep(name = "xla", version = "20241025.0-4663f04") +bazel_dep(name = "xla", version = "20241025.1-4663f04") tsl = use_extension("@xla//:tsl.bzl", "tsl") use_repo(tsl, "tsl") diff --git a/examples/benchmark/BUILD.bazel b/examples/benchmark/BUILD.bazel index 7ca74ada..f4cffe42 100644 --- a/examples/benchmark/BUILD.bazel +++ b/examples/benchmark/BUILD.bazel @@ -6,6 +6,7 @@ zig_cc_binary( deps = [ "@zml//async", "@zml//zml", + "@zml//ffi", "//third_party/tigerbeetle:flags", ], ) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index ee032fc4..de4b9f2e 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -2,6 +2,7 @@ const std = @import("std"); const zml = @import("zml"); const asynk = @import("async"); const flags = @import("tigerbeetle/flags"); +const ffi = @import("ffi"); // set log level to debug to print the generated IR pub const std_options = .{ @@ -12,7 +13,11 @@ pub const std_options = .{ const Benchmark = struct { pub fn forward(self: Benchmark, a: zml.Tensor, b: zml.Tensor) zml.Tensor { _ = self; - return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); + var c = a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); + + c = c.print(); + + return c; } }; @@ -72,7 +77,7 @@ pub fn asyncMain() !void { var executable = try compiled.prepare(arena, .{}); defer executable.deinit(); - var rng = std.Random.DefaultPrng.init(0); + var rng = std.Random.DefaultPrng.init(1); const random = rng.random(); var a_buffer = try createRandomBuffer(allocator, platform, a_shape, random); diff --git a/ffi/BUILD.bazel b/ffi/BUILD.bazel new file mode 100644 index 00000000..e8555e35 --- /dev/null +++ b/ffi/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "ffi", + #srcs = ["ffi.zig"], + main = "ffi.zig", + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "@xla//xla/ffi/api:c_api", + ], +) + diff --git a/ffi/ffi.zig b/ffi/ffi.zig new file mode 100644 index 00000000..9e144135 --- /dev/null +++ b/ffi/ffi.zig @@ -0,0 +1,466 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +pub const c = @import("c"); + +const log = std.log.scoped(.ffi); + +fn ffiStructSize(comptime T: type) usize { + // unsafe on purpose, we want this to fail if that ever changes + const typedef_name = comptime blk: { + const needle = ".struct_"; + const idx = std.mem.indexOf(u8, @typeName(T), needle).?; + break :blk @typeName(T)[idx + needle.len ..]; + }; + return @field(c, typedef_name ++ "_STRUCT_SIZE"); +} + +pub inline fn ffiStruct(v: anytype) @TypeOf(v) { + var ret = v; + ret.struct_size = ffiStructSize(@TypeOf(v)); + return ret; +} + +pub const ApiVersion = extern struct { + pub const major = c.XLA_FFI_API_MAJOR; + pub const minor = c.XLA_FFI_API_MINOR; + + struct_size: usize, + extension_start: ?*ExtensionBase, + major_version: i32, + minor_version: i32, +}; + +pub const ExtensionType = enum(c.XLA_FFI_Extension_Type) { + metadata = c.XLA_FFI_Extension_Metadata, +}; + +pub const ExtensionBase = extern struct { + struct_size: usize, + type: ExtensionType, + next: ?*ExtensionBase, +}; + +// TODO(Corentin): This is actually a bit set. +pub const HandlerTraits = u32; + +pub const Metadata = extern struct { + struct_size: usize, + api_version: ApiVersion, + traits: HandlerTraits, +}; + +pub const MetadataExtension = extern struct { + extension_base: ExtensionBase, + metadata: ?*Metadata, +}; + +pub const ApiError = error{ + Cancelled, + Unknown, + InvalidArgument, + DeadlineExceeded, + NotFound, + AlreadyExists, + PermissionDenied, + ResourceExhausted, + FailedPrecondition, + Aborted, + OutOfRange, + Unimplemented, + Internal, + Unavailable, + DataLoss, + Unauthenticated, +}; + +fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { + return struct { + pub fn to(self: anytype) switch (@TypeOf(self)) { + *T => *InnerT, + *const T => *const InnerT, + else => unreachable, + } { + return @ptrCast(@alignCast(self)); + } + + pub fn from(self: anytype) switch (@TypeOf(self)) { + *InnerT => *T, + *const InnerT => *const T, + else => unreachable, + } { + return @ptrCast(@alignCast(self)); + } + }; +} + +pub const Api = opaque { + pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; + + pub fn getStream(self: *const Api, context: ?*ExecutionContext) ApiError!*anyopaque { + var ret = ffiStruct(c.XLA_FFI_Stream_Get_Args{ + .ctx = if (context) |ctx| ctx.inner() else null, + }); + const result = self.inner().XLA_FFI_Stream_Get.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(self); + log.err("[Api.getStream] {s}", .{err.getMessage(self)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + return ret.stream.?; + } + + pub fn allocateDeviceMemory(self: *const Api, context: ?*ExecutionContext, size: usize, alignment: usize) ApiError!*anyopaque { + var ret = ffiStruct(c.XLA_FFI_DeviceMemory_Allocate_Args{ + .ctx = if (context) |ctx| ctx.inner() else null, + .size = size, + .alignment = alignment, + }); + const result = self.inner().XLA_FFI_DeviceMemory_Allocate.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(self); + log.err("[Api.allocateDeviceMemory] {s}", .{err.getMessage(self)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + return ret.data.?; + } + + pub fn freeDeviceMemory(self: *const Api, context: ?*ExecutionContext, data: *anyopaque, size: usize) ApiError!void { + var ret = ffiStruct(c.XLA_FFI_DeviceMemory_Free_Args{ + .ctx = if (context) |ctx| ctx.inner() else null, + .size = size, + .data = data, + }); + const result = self.inner().XLA_FFI_DeviceMemory_Free.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(self); + log.err("[Api.freeDeviceMemory] {s}", .{err.getMessage(self)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } + + // TODO(Corentin): Implement remaining methods if needed: + // * XLA_FFI_ThreadPool_Schedule + // * XLA_FFI_Handler_Register + // * XLA_FFI_TypeId_Register + // * XLA_FFI_State_Set + // * XLA_FFI_State_Get +}; + +pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { + instantiate = c.XLA_FFI_ExecutionStage_INSTANTIATE, + prepare = c.XLA_FFI_ExecutionStage_PREPARE, + initialize = c.XLA_FFI_ExecutionStage_INITIALIZE, + execute = c.XLA_FFI_ExecutionStage_EXECUTE, +}; + +pub const ExecutionContext = opaque { + pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; + + pub fn get(self: *ExecutionContext, api: *const Api, type_id: *TypeId) ApiError!*anyopaque { + var ret = ffiStruct(c.XLA_FFI_ExecutionContext_Get_Args{ + .ctx = self.inner(), + .type_id = @ptrCast(@alignCast(type_id)), + }); + const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(self); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(self)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + return ret.data.?; + } +}; + +pub const TypeId = extern struct { + type_id: i64, +}; + +pub const ArgType = enum(c.XLA_FFI_ArgType) { + buffer = c.XLA_FFI_ArgType_BUFFER, +}; + +pub const RetType = enum(c.XLA_FFI_RetType) { + buffer = c.XLA_FFI_RetType_BUFFER, +}; + +pub const Arg = union(ArgType) { + buffer: Buffer, +}; + +pub const Ret = union(ArgType) { + buffer: Buffer, +}; + +pub const DataType = enum(c.XLA_FFI_DataType) { + invalid = c.XLA_FFI_DataType_INVALID, + pred = c.XLA_FFI_DataType_PRED, + s8 = c.XLA_FFI_DataType_S8, + s16 = c.XLA_FFI_DataType_S16, + s32 = c.XLA_FFI_DataType_S32, + s64 = c.XLA_FFI_DataType_S64, + u8 = c.XLA_FFI_DataType_U8, + u16 = c.XLA_FFI_DataType_U16, + u32 = c.XLA_FFI_DataType_U32, + u64 = c.XLA_FFI_DataType_U64, + f16 = c.XLA_FFI_DataType_F16, + f32 = c.XLA_FFI_DataType_F32, + f64 = c.XLA_FFI_DataType_F64, + bf16 = c.XLA_FFI_DataType_BF16, + c64 = c.XLA_FFI_DataType_C64, + c128 = c.XLA_FFI_DataType_C128, + token = c.XLA_FFI_DataType_TOKEN, + f8e5m2 = c.XLA_FFI_DataType_F8E5M2, + f8e3m4 = c.XLA_FFI_DataType_F8E3M4, + f8e4m3 = c.XLA_FFI_DataType_F8E4M3, + f8e4m3fn = c.XLA_FFI_DataType_F8E4M3FN, + f8e4m3b11fnuz = c.XLA_FFI_DataType_F8E4M3B11FNUZ, + f8e5m2fnuz = c.XLA_FFI_DataType_F8E5M2FNUZ, + f8e4m3fnuz = c.XLA_FFI_DataType_F8E4M3FNUZ, +}; + +pub const Buffer = extern struct { + struct_size: usize, + extension_start: ?*c.XLA_FFI_Extension_Base, + dtype: DataType, + data: ?*anyopaque, + rank: i64, + dims: [*]i64, +}; + +pub const Args = extern struct { + struct_size: usize, + extension_start: ?*c.XLA_FFI_Extension_Base, + size: i64, + types: [*]ArgType, + args: [*]?*anyopaque, + + pub fn getArgAs(self: Args, comptime T: type, index: usize) *T { + if (index > @as(usize, @intCast(self.size))) @panic("Index out of bound"); + return @ptrCast(@alignCast(self.args[index].?)); + } +}; + +pub const Rets = extern struct { + struct_size: usize, + extension_start: ?*c.XLA_FFI_Extension_Base, + size: i64, + types: [*]ArgType, + rets: [*]?*anyopaque, + + pub fn getRetAs(self: Rets, comptime T: type, index: usize) *T { + if (index > @as(usize, @intCast(self.size))) @panic("Index out of bound"); + return @ptrCast(@alignCast(self.rets[index].?)); + } +}; + +pub const AttrType = enum(c.XLA_FFI_AttrType) { + array = c.XLA_FFI_AttrType_ARRAY, + dictionary = c.XLA_FFI_AttrType_DICTIONARY, + scalar = c.XLA_FFI_AttrType_SCALAR, + string = c.XLA_FFI_AttrType_STRING, +}; + +pub const ByteSpan = extern struct { + ptr: [*]const u8, + len: usize, + + pub fn slice(self: ByteSpan) []const u8 { + return self.ptr[0..self.len]; + } +}; + +pub const Scalar = extern struct { + dtype: DataType, + value: *anyopaque, +}; + +pub const Array = extern struct { + dtype: DataType, + size: usize, + data: [*]anyopaque, +}; + +pub const Attrs = extern struct { + struct_size: usize, + extension_start: ?*ExtensionBase, + size: i64, + types: [*]AttrType, + names: [*]*ByteSpan, + attrs: [*]?*anyopaque, + + pub fn getAttrAs(self: Attrs, comptime T: type, index: usize) *T { + if (index > @as(usize, @intCast(self.size))) @panic("Index out of bound"); + return @ptrCast(@alignCast(self.attrs[index].?)); + } + + pub fn getName(self: Attrs, index: usize) []const u8 { + if (index > @as(usize, @intCast(self.size))) @panic("Index out of bound"); + const span = self.names[index].*; + return span.ptr[0..span.len]; + } + + pub fn getAttrByNameAs(self: Attrs, comptime T: type, name: []const u8) ?*T { + const index: usize = for (0..@intCast(self.size)) |index| { + const attr_name = self.names[index].slice(); + if (std.mem.eql(u8, attr_name, name)) break index; + } else return null; + + return self.getAttrAs(T, index); + } +}; + +pub const CallFrame = extern struct { + struct_size: usize, + extension_start: ?*ExtensionBase, + api: ?*const Api, + ctx: ?*ExecutionContext, + stage: ExecutionStage, + args: Args, + rets: Rets, + attrs: Attrs, + future: ?*Future, +}; + +pub const Handler = fn (*CallFrame) callconv(.C) ?*Error; + +pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { + cancelled = c.XLA_FFI_Error_Code_CANCELLED, + unknown = c.XLA_FFI_Error_Code_UNKNOWN, + invalid_argument = c.XLA_FFI_Error_Code_INVALID_ARGUMENT, + deadline_exceeded = c.XLA_FFI_Error_Code_DEADLINE_EXCEEDED, + not_found = c.XLA_FFI_Error_Code_NOT_FOUND, + already_exists = c.XLA_FFI_Error_Code_ALREADY_EXISTS, + permission_denied = c.XLA_FFI_Error_Code_PERMISSION_DENIED, + resource_exhausted = c.XLA_FFI_Error_Code_RESOURCE_EXHAUSTED, + failed_precondition = c.XLA_FFI_Error_Code_FAILED_PRECONDITION, + aborted = c.XLA_FFI_Error_Code_ABORTED, + out_of_range = c.XLA_FFI_Error_Code_OUT_OF_RANGE, + unimplemented = c.XLA_FFI_Error_Code_UNIMPLEMENTED, + internal = c.XLA_FFI_Error_Code_INTERNAL, + unavailable = c.XLA_FFI_Error_Code_UNAVAILABLE, + data_loss = c.XLA_FFI_Error_Code_DATA_LOSS, + unauthenticated = c.XLA_FFI_Error_Code_UNAUTHENTICATED, + + pub fn toApiError(code: ErrorCode) ApiError { + return switch (code) { + .cancelled => error.Cancelled, + .unknown => error.Unknown, + .invalid_argument => error.InvalidArgument, + .deadline_exceeded => error.DeadlineExceeded, + .not_found => error.NotFound, + .already_exists => error.AlreadyExists, + .permission_denied => error.PermissionDenied, + .resource_exhausted => error.ResourceExhausted, + .failed_precondition => error.FailedPrecondition, + .aborted => error.Aborted, + .out_of_range => error.OutOfRange, + .unimplemented => error.Unimplemented, + .internal => error.Internal, + .unavailable => error.Unavailable, + .data_loss => error.DataLoss, + .unauthenticated => error.Unauthenticated, + }; + } +}; + +pub const Error = opaque { + pub const inner = TransmuteMixin(Error, c.XLA_FFI_Error).to; + pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; + + pub fn create(api: *const Api, error_code: ErrorCode, message: [:0]const u8) *Error { + var ret = ffiStruct(c.XLA_FFI_Error_Create_Args{ + .message = message.ptr, + .errc = @intFromEnum(error_code), + }); + return fromInner(api.inner().XLA_FFI_Error_Create.?(&ret).?); + } + + pub fn destroy(err: *Error, api: *const Api) void { + var ret = ffiStruct(c.XLA_FFI_Error_Destroy_Args{ .@"error" = err.inner() }); + api.inner().XLA_FFI_Error_Destroy.?(&ret); + } + + pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { + var ret = ffiStruct(c.XLA_FFI_Error_GetMessage_Args{ + .@"error" = err.inner(), + }); + api.inner().XLA_FFI_Error_GetMessage.?(&ret); + return std.mem.span(ret.message); + } +}; + +pub const Future = opaque { + pub const inner = TransmuteMixin(Future, c.XLA_FFI_Future).to; + pub const fromInner = TransmuteMixin(Future, c.XLA_FFI_Future).from; + + pub fn create(api: *const Api) ApiError!*Future { + var ret = ffiStruct(c.XLA_FFI_Future_Create_Args{}); + const result = api.inner().XLA_FFI_Future_Create.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[Future.create] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + return fromInner(ret.future.?); + } + + pub fn setAvailable(self: *Future, api: *const Api) ApiError!void { + var ret = ffiStruct(c.XLA_FFI_Future_SetAvailable_Args{ + .future = self.inner(), + }); + + const result = api.inner().XLA_FFI_Future_SetAvailable.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[Future.setAvailable] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } + + pub fn setError(self: *Future, api: *const Api, err: *Error) ApiError!void { + var ret = ffiStruct(c.XLA_FFI_Future_SetError_Args{ + .future = self.inner(), + .@"error" = err.inner(), + }); + + const result = api.inner().XLA_FFI_Future_SetError.?(&ret); + + if (result) |ffi_error| { + const err2 = Error.fromInner(ffi_error); + defer err2.destroy(api); + log.err("[Future.setError] {s}", .{err2.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } +}; diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 847e60aa..7a635e97 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -29,6 +29,7 @@ zig_library( main = "stablehlo.zig", visibility = ["//mlir/dialects:__subpackages__"], deps = [ + "//stdx", "//mlir", "//mlir:c", "@stablehlo//:stablehlo_capi", diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index 6d2c7400..1dfe23d6 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const stdx = @import("stdx"); const c = @import("c"); const mlir = @import("mlir"); @@ -732,32 +733,159 @@ pub fn convolution( }); } -pub const CustomCallOpts = struct { - call_target_name: [:0]const u8, - has_side_effect: bool, - backend_config: [:0]const u8 = &.{}, - api_version: i32, - output_operand_aliases: []const i64, +pub const BackendConfig = struct { + entries: []const struct { name: [:0]const u8, value: BackendConfigValue } = &.{}, + + pub fn toMlirAttribute(self: BackendConfig, allocator: std.mem.Allocator, ctx: mlir.Context) error{OutOfMemory}!mlir.Attribute { + const named_attributes = try allocator.alloc(mlir.NamedAttribute, self.entries.len); + errdefer allocator.free(named_attributes); + + for (self.entries, named_attributes) |entry, *named_attribute| { + named_attribute.* = mlir.NamedAttribute.init(mlir.Identifier.get(ctx, entry.name), try entry.value.toMlirAttribute(allocator, ctx)); + } + + return mlir.DictionaryAttribute.init(ctx, named_attributes).as(mlir.Attribute).?; + } +}; + +pub const ScalarType = enum { + i64, + f64, +}; + +pub const Scalar = union(ScalarType) { + i64: i64, + f64: f64, + + pub fn toMlirAttribute(self: Scalar, ctx: mlir.Context) !mlir.Attribute { + return switch (self) { + .i64 => |v| mlir.IntegerAttribute(.i64).init(ctx, v).as(mlir.Attribute).?, + .f64 => |v| mlir.FloatAttribute(.f64).init(ctx, v).as(mlir.Attribute).?, + }; + } +}; + +pub const ScalarArray = union(ScalarType) { + i64: []const i64, + f64: []const f64, + + pub fn toMlirAttribute(self: ScalarArray, allocator: std.mem.Allocator, ctx: mlir.Context) !mlir.Attribute { + return switch (self) { + .i64 => |v| b: { + const attributes = try allocator.alloc(mlir.Attribute, v.len); + errdefer allocator.free(attributes); + + for (attributes, v) |*a, v2| a.* = mlir.IntegerAttribute(.i64).init(ctx, v2).as(mlir.Attribute).?; + break :b mlir.ArrayAttribute.init(ctx, attributes).as(mlir.Attribute).?; + }, + .f64 => |v| b: { + const attributes = try allocator.alloc(mlir.Attribute, v.len); + errdefer allocator.free(attributes); + + for (attributes, v) |*a, v2| a.* = mlir.FloatAttribute(.f64).init(ctx, v2).as(mlir.Attribute).?; + break :b mlir.ArrayAttribute.init(ctx, attributes).as(mlir.Attribute).?; + }, + }; + } +}; + +pub const BackendConfigValueType = enum { + scalar, + array, + dictionary, + string, +}; + +pub const BackendConfigValue = union(BackendConfigValueType) { + scalar: Scalar, + array: ScalarArray, + dictionary: BackendConfig, + string: []const u8, + + pub fn toMlirAttribute(self: BackendConfigValue, allocator: std.mem.Allocator, ctx: mlir.Context) !mlir.Attribute { + return switch (self) { + .scalar => |scalar| scalar.toMlirAttribute(ctx), + .array => |array| array.toMlirAttribute(allocator, ctx), + .dictionary => |dictionary| dictionary.toMlirAttribute(allocator, ctx), + .string => |string| mlir.StringAttribute.init(ctx, string).as(mlir.Attribute).?, + }; + } +}; + +pub const CustomCallType = enum { + legacy, + ffi, +}; + +pub const CustomCallOpts = union(CustomCallType) { + legacy: struct { + api_version: i64 = 0, + call_target_name: [:0]const u8, + has_side_effect: bool, + backend_config: [:0]const u8 = &.{}, + output_operand_aliases: []const i64, + }, + ffi: struct { + api_version: i64 = 4, + call_target_name: [:0]const u8, + has_side_effect: bool, + backend_config: BackendConfig = .{}, + output_operand_aliases: []const i64, + }, + + pub fn callTargetName(self: CustomCallOpts) [:0]const u8 { + return switch (self) { + inline else => |v| v.call_target_name, + }; + } + + pub fn hasSideEffect(self: CustomCallOpts) bool { + return switch (self) { + inline else => |v| v.has_side_effect, + }; + } + + pub fn outputOperandAlias(self: CustomCallOpts) []const i64 { + return switch (self) { + inline else => |v| v.output_operand_aliases, + }; + } + + pub fn apiVersion(self: CustomCallOpts) i64 { + return switch (self) { + inline else => |v| v.api_version, + }; + } }; pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { + switch (opts) { + .legacy => |v| stdx.debug.assert(v.api_version >= 0 and v.api_version <= 3, "Expected api_version to be 0, 1, 2 or 3 for legacy custom call, got {}", .{v.api_version}), + .ffi => |v| stdx.debug.assert(v.api_version == 4, "Expected api_version to be 4 for ffi custom call, got {}", .{v.api_version}), + } + var buffer: [1024]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&buffer); const allocator = fba.allocator(); - const output_operand_aliases = allocator.alloc(mlir.Attribute, opts.output_operand_aliases.len) catch unreachable; - for (opts.output_operand_aliases, 0..) |alias, i| { + const output_operand_aliases = allocator.alloc(mlir.Attribute, opts.outputOperandAlias().len) catch unreachable; + for (opts.outputOperandAlias(), 0..) |alias, i| { output_operand_aliases[i] = OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).as(mlir.Attribute).?; } + const backend_config_attribute = switch (opts) { + .legacy => |v| mlir.StringAttribute.init(ctx, v.backend_config).as(mlir.Attribute).?, + .ffi => |v| v.backend_config.toMlirAttribute(allocator, ctx) catch unreachable, + }; + return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ .operands = inputs, .results = res_types, .attributes = &.{ - .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, opts.api_version).as(mlir.Attribute).? }, - .{ "call_target_name", mlir.StringAttribute.init(ctx, opts.call_target_name).as(mlir.Attribute).? }, - .{ "has_side_effect", mlir.BoolAttribute.init(ctx, opts.has_side_effect).as(mlir.Attribute).? }, - .{ "backend_config", mlir.StringAttribute.init(ctx, opts.backend_config).as(mlir.Attribute).? }, + .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, opts.apiVersion()).as(mlir.Attribute).? }, + .{ "call_target_name", mlir.StringAttribute.init(ctx, opts.callTargetName()).as(mlir.Attribute).? }, + .{ "has_side_effect", mlir.BoolAttribute.init(ctx, opts.hasSideEffect()).as(mlir.Attribute).? }, + .{ "backend_config", backend_config_attribute }, .{ "output_operand_aliases", mlir.ArrayAttribute.init(ctx, output_operand_aliases).as(mlir.Attribute).? }, }, .location = location, diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 7dc59277..b61715c6 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -16,6 +16,7 @@ zig_library( deps = [ ":profiler_options_proto", "//stdx", + "//ffi", "@xla//xla/pjrt/c:pjrt_c_api_gpu_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_profiler_extension_hdrs", diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index cee62af3..84a6fd2f 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1,6 +1,7 @@ const builtin = @import("builtin"); const std = @import("std"); const stdx = @import("stdx"); +const ffi = @import("ffi"); const c = @import("c"); @@ -909,11 +910,26 @@ pub const CustomCall = fn (*anyopaque, [*]*anyopaque, [*]const u8, usize) callco pub const CustomCallRegistry = extern struct { inner: *const c.PJRT_Gpu_Register_Custom_Call, - pub fn register(self: *const CustomCallRegistry, api: *const Api, api_version: usize, name: []const u8, func: *const CustomCall) ApiError!void { + pub fn registerLegacy(self: *const CustomCallRegistry, api: *const Api, name: []const u8, func: *const CustomCall) ApiError!void { var ret = pjrtStruct(c.PJRT_Gpu_Register_Custom_Call_Args{ .function_name = name.ptr, .function_name_size = name.len, - .api_version = @intCast(api_version), + .api_version = 0, + .handler_execute = @ptrCast(@constCast(func)), + }); + const result = self.inner(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + log.err("[GpuRegisterCustomCall] {s}", .{pjrt_error.getMessage(api)}); + return pjrt_error.getCode(api).toApiError(); + } + } + + pub fn registerFfi(self: *const CustomCallRegistry, api: *const Api, name: []const u8, func: *const ffi.Handler) ApiError!void { + var ret = pjrtStruct(c.PJRT_Gpu_Register_Custom_Call_Args{ + .function_name = name.ptr, + .function_name_size = name.len, + .api_version = 1, .handler_execute = @ptrCast(@constCast(func)), }); const result = self.inner(&ret); diff --git a/third_party/modules/xla/20241025.1-4663f04/MODULE.bazel b/third_party/modules/xla/20241025.1-4663f04/MODULE.bazel new file mode 100644 index 00000000..fe5648f8 --- /dev/null +++ b/third_party/modules/xla/20241025.1-4663f04/MODULE.bazel @@ -0,0 +1,34 @@ +module( + name = "xla", + version = "20241025.1-4663f04", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +bazel_dep(name = "stablehlo", version = "20241021.0-1c0b606") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", +) diff --git a/third_party/modules/xla/20241025.1-4663f04/overlay/MODULE.bazel b/third_party/modules/xla/20241025.1-4663f04/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20241025.1-4663f04/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20241025.1-4663f04/overlay/tsl.bzl b/third_party/modules/xla/20241025.1-4663f04/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20241025.1-4663f04/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20241025.1-4663f04/overlay/workspace.bzl b/third_party/modules/xla/20241025.1-4663f04/overlay/workspace.bzl new file mode 100644 index 00000000..87c035b7 --- /dev/null +++ b/third_party/modules/xla/20241025.1-4663f04/overlay/workspace.bzl @@ -0,0 +1,52 @@ +load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "@tsl//third_party/grpc:generate_cc_env_fix.patch", + "@tsl//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", + "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20241025.1-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20241025.1-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch new file mode 100644 index 00000000..0f58f107 --- /dev/null +++ b/third_party/modules/xla/20241025.1-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch @@ -0,0 +1,27 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +index c821916ad..89a596123 100644 +--- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +@@ -16,8 +16,10 @@ limitations under the License. + #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + ++#ifdef __cplusplus + #include + #include ++#endif + + #include "xla/backends/profiler/plugin/profiler_c_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/xla/20241025.1-4663f04/patches/0004-FFI-Valid-C-Api.patch b/third_party/modules/xla/20241025.1-4663f04/patches/0004-FFI-Valid-C-Api.patch new file mode 100644 index 00000000..09fdca86 --- /dev/null +++ b/third_party/modules/xla/20241025.1-4663f04/patches/0004-FFI-Valid-C-Api.patch @@ -0,0 +1,58 @@ +diff --git a/xla/ffi/api/c_api.h b/xla/ffi/api/c_api.h +index f0c4f40e78..7fdda0a52d 100644 +--- a/xla/ffi/api/c_api.h ++++ b/xla/ffi/api/c_api.h +@@ -255,30 +255,30 @@ typedef struct XLA_FFI_ExecutionContext XLA_FFI_ExecutionContext; + //===----------------------------------------------------------------------===// + + // TypeId uniquely identifies a user-defined type in a given XLA FFI instance. +-struct XLA_FFI_TypeId { ++typedef struct XLA_FFI_TypeId { + int64_t type_id; +-}; ++} XLA_FFI_TypeId; + + // We use byte spans to pass strings to handlers because strings might not be + // null terminated, and even if they are, looking for a null terminator can + // become very expensive in tight loops. +-struct XLA_FFI_ByteSpan { ++typedef struct XLA_FFI_ByteSpan { + const char* ptr; + size_t len; +-}; ++} XLA_FFI_ByteSpan; + + // A struct to pass a scalar value to FFI handler. +-struct XLA_FFI_Scalar { ++typedef struct XLA_FFI_Scalar { + XLA_FFI_DataType dtype; + void* value; +-}; ++} XLA_FFI_Scalar; + + // A struct to pass a dense array to FFI handler. +-struct XLA_FFI_Array { ++typedef struct XLA_FFI_Array { + XLA_FFI_DataType dtype; + size_t size; + void* data; +-}; ++} XLA_FFI_Array; + + //===----------------------------------------------------------------------===// + // Future +@@ -431,12 +431,12 @@ XLA_FFI_DEFINE_STRUCT_TRAITS(XLA_FFI_CallFrame, attrs); + typedef XLA_FFI_Error* XLA_FFI_Handler(XLA_FFI_CallFrame* call_frame); + + // XLA FFI handlers for execution stages (see XLA_FFI_ExecutionStage). +-struct XLA_FFI_Handler_Bundle { ++typedef struct XLA_FFI_Handler_Bundle { + XLA_FFI_Handler* instantiate; // optional + XLA_FFI_Handler* prepare; // optional + XLA_FFI_Handler* initialize; // optional + XLA_FFI_Handler* execute; // required +-}; ++} XLA_FFI_Handler_Bundle; + + enum XLA_FFI_Handler_TraitsBits { + // Calls to FFI handler are safe to trace into the command buffer. It means diff --git a/third_party/modules/xla/20241025.1-4663f04/source.json b/third_party/modules/xla/20241025.1-4663f04/source.json new file mode 100644 index 00000000..69039a0f --- /dev/null +++ b/third_party/modules/xla/20241025.1-4663f04/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "xla-4663f04d2653d686833f9c306bd0d899c3127358", + "url": "https://github.com/openxla/xla/archive/4663f04d2653d686833f9c306bd0d899c3127358.tar.gz", + "integrity": "sha256-tQRn0TXkPcmKpCcIuTYM10XNB1qC/d6++4rA1mlnnKw=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "", + "0004-FFI-Valid-C-Api.patch": "" + } +} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 39a8fd78..4d504722 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -31,6 +31,7 @@ zig_library( "//mlir", "//mlir/dialects", "//pjrt", + "//ffi", "//runtimes", "//stdx", "//zml/tools", diff --git a/zml/context.zig b/zml/context.zig index 0255d49b..a82e21fd 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -8,6 +8,7 @@ const stdx = @import("stdx"); const platform = @import("platform.zig"); const pjrt = @import("pjrtx.zig"); +const ffi = @import("ffi"); const HostBuffer = @import("hostbuffer.zig").HostBuffer; const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); @@ -194,7 +195,7 @@ const cuda = struct { cuda.runtime = try Runtime.init(); const registry = cuda_platform.pjrt_api.customCallRegistry().?; - try registry.register(cuda_platform.pjrt_api, 0, "zmlHostBufferCallback", &hostBufferCallback); + try registry.registerFfi(cuda_platform.pjrt_api, "zmlHostBufferCallback", &hostBufferCallback); } pub const Stream = opaque {}; @@ -224,21 +225,29 @@ const cuda = struct { } }; - fn getContext(args: [*]const u8, args_len: usize) struct { *const Context.HostCallback, *Context.HostCallbackCtx } { - std.debug.assert(args_len == @sizeOf(*anyopaque) * 2); + fn getContext(attrs: ffi.Attrs) struct { *const Context.HostCallback, *Context.HostCallbackCtx } { + const context_scalar = attrs.getAttrByNameAs(ffi.Scalar, "context") orelse unreachable; + std.debug.assert(context_scalar.dtype == .s64); + const ctx: *Context.HostCallbackCtx = @ptrFromInt(@as(usize, @bitCast(@as(*i64, @ptrCast(@alignCast(context_scalar.value))).*))); - const raw_fn_ptr: usize = @bitCast(args[0..@sizeOf(*anyopaque)].*); - const fn_ptr: *const Context.HostCallback = @ptrFromInt(raw_fn_ptr); + const callback_scalar = attrs.getAttrByNameAs(ffi.Scalar, "callback") orelse unreachable; + std.debug.assert(callback_scalar.dtype == .s64); + const callback: *const Context.HostCallback = @ptrFromInt(@as(usize, @bitCast(@as(*i64, @ptrCast(@alignCast(callback_scalar.value))).*))); - const raw_ctx_ptr: usize = @bitCast(args[@sizeOf(*anyopaque)..][0..@sizeOf(*anyopaque)].*); - const ctx_ptr: *Context.HostCallbackCtx = @ptrFromInt(raw_ctx_ptr); - return .{ fn_ptr, ctx_ptr }; + return .{ callback, ctx }; } - fn hostBufferCallback(opaque_stream: *anyopaque, buffers: [*]*anyopaque, args: [*]const u8, args_len: usize) callconv(.C) void { - const stream: *Stream = @ptrCast(opaque_stream); - const src: *anyopaque = buffers[0]; - const callback, const ctx = getContext(args, args_len); + fn hostBufferCallback(call_frame: *ffi.CallFrame) callconv(.C) ?*ffi.Error { + if (call_frame.extension_start != null and call_frame.extension_start.?.type == .metadata) { + const metadata_extension: *ffi.MetadataExtension = @fieldParentPtr("extension_base", call_frame.extension_start.?); + metadata_extension.metadata.?.api_version.major_version = ffi.ApiVersion.major; + metadata_extension.metadata.?.api_version.minor_version = ffi.ApiVersion.minor; + return null; + } + + const stream: *Stream = @ptrCast(call_frame.api.?.getStream(call_frame.ctx.?) catch unreachable); + const src: *anyopaque = call_frame.args.getArgAs(ffi.Buffer, 0).data.?; + const callback, const ctx = getContext(call_frame.attrs); // Add synchronization because this is called from the device driver. ctx.mutex.lock(); @@ -251,5 +260,6 @@ const cuda = struct { _ = synchronize_result; callback(ctx.host); + return null; } }; diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index f00ca762..6e8d55c0 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -127,13 +127,13 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const op = dialect.stablehlo.custom_call( ctx.mlirCtx(), &.{ q.value(), k.value(), v.value(), bias.value() }, - .{ + .{ .legacy = .{ .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", .backend_config = backend_config, .api_version = 2, .has_side_effect = false, .output_operand_aliases = &.{}, - }, + } }, &.{ mlir.ext.mlirType(ctx.mlirCtx(), q.shape()), mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(ctx.mlirCtx()).as(mlir.Type).?).as(mlir.Type).?, diff --git a/zml/ops.zig b/zml/ops.zig index aa170382..88d098e8 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -683,10 +683,9 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base /// /// For example, this can be used to extract tokens quickly if they run on a loop on the /// GPU. -pub fn identityCustomCall(name: [:0]const u8, input: Tensor, context: *anyopaque) Tensor { - const address: [8]u8 = @bitCast(@intFromPtr(context)); - var backend_config: [8:0]u8 = undefined; - @memcpy(backend_config[0..8], address[0..8]); +pub fn identityCustomCall(name: [:0]const u8, input: Tensor, context: ?*anyopaque) Tensor { + const context_ptr: i64 = @bitCast(@intFromPtr(context)); + const ctx = CompilationContext.current(); const loc = ctx.mlirCtx().location(@src()).namedFmt(ctx.mlirCtx(), "custom_call({s})", .{name}); @@ -694,13 +693,13 @@ pub fn identityCustomCall(name: [:0]const u8, input: Tensor, context: *anyopaque const op = dialect.stablehlo.custom_call( ctx.mlirCtx(), &.{input.value()}, - .{ - .api_version = 1, + .{ .ffi = .{ + .api_version = 4, .has_side_effect = false, .call_target_name = name, - .backend_config = backend_config[0..], + .backend_config = .{ .entries = &.{.{ .name = "context", .value = .{ .scalar = .{ .i64 = context_ptr } } }} }, .output_operand_aliases = &.{0}, - }, + } }, &.{input.value().getType()}, loc, ); @@ -733,20 +732,22 @@ pub fn addHostCallback( .host = HostBuffer.fromBytes(input.shape(), full_data[0..len]), }; - const backend_config: [2:null]?*const anyopaque = .{ callback, stable_ctx_ptr }; const ctx = CompilationContext.current(); const loc = ctx.mlirCtx().location(@src()); const op = dialect.stablehlo.custom_call( ctx.mlirCtx(), &.{input.value()}, - .{ - .api_version = 1, + .{ .ffi = .{ + .api_version = 4, .has_side_effect = false, .call_target_name = "zmlHostBufferCallback", - .backend_config = @ptrCast(std.mem.sliceAsBytes(&backend_config)), + .backend_config = .{ .entries = &.{ + .{ .name = "callback", .value = .{ .scalar = .{ .i64 = @bitCast(@intFromPtr(callback)) } } }, + .{ .name = "context", .value = .{ .scalar = .{ .i64 = @bitCast(@intFromPtr(stable_ctx_ptr)) } } }, + } }, .output_operand_aliases = &.{0}, - }, + } }, &.{input.value().getType()}, loc, );