From ffbb293331fe6dada30eac8eda7f9fba32da445d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 27 May 2025 17:20:58 +0200 Subject: [PATCH 1/4] zml: custom call v2 --- examples/custom_call/BUILD | 11 ++ examples/custom_call/main.zig | 154 +++++++++++++++++++++++ pjrt/ffi.zig | 2 +- pjrt/pjrt.zig | 8 +- zml/BUILD.bazel | 1 + zml/buffer.zig | 46 +++++-- zml/context.zig | 69 +---------- zml/custom_call.zig | 224 ++++++++++++++++++++++++++++++++++ zml/exe.zig | 38 ++++++ zml/hostbuffer.zig | 10 ++ zml/module.zig | 8 +- zml/ops.zig | 27 ---- zml/pjrtx.zig | 7 ++ zml/tensor.zig | 19 +-- zml/zml.zig | 1 + 15 files changed, 494 insertions(+), 131 deletions(-) create mode 100644 examples/custom_call/BUILD create mode 100644 examples/custom_call/main.zig create mode 100644 zml/custom_call.zig diff --git a/examples/custom_call/BUILD b/examples/custom_call/BUILD new file mode 100644 index 00000000..a2aa32fb --- /dev/null +++ b/examples/custom_call/BUILD @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "custom_call", + copts = ["-freference-trace=30"], + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig new file mode 100644 index 00000000..e5a17443 --- /dev/null +++ b/examples/custom_call/main.zig @@ -0,0 +1,154 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +const cuda = zml.context.cuda; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +fn add_op_host_func(data: *anyopaque) callconv(.c) void { + const ctx = @as(*AddOp, @alignCast(@ptrCast(data))); + + const result = std.mem.bytesAsValue(f32, ctx.result.asHostBuffer().mutBytes()); + result.* = ctx.a.asHostBuffer().item(f32, 0) + ctx.b.asHostBuffer().item(f32, 0); +} + +pub const AddOp = struct { + pub var type_id: i64 = undefined; + const Self = @This(); + + allocator: std.mem.Allocator, + platform: zml.Platform, + + a: zml.Buffer, + b: zml.Buffer, + result: zml.Buffer, + + results: [1]zml.Buffer = undefined, + stream: *anyopaque = undefined, + + pub fn init( + allocator: std.mem.Allocator, + platform: zml.Platform, + io_shape: zml.Shape, + ) !AddOp { + const host_buffer = try zml.HostBuffer.empty(allocator, io_shape); + defer host_buffer.deinit(allocator); + + const a = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + const b = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + const result = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + + return .{ + .a = a, + .b = b, + .result = result, + .platform = platform, + .allocator = allocator, + }; + } + + pub fn deinit(self: *Self) void { + self.a.deinit(); + self.b.deinit(); + self.result.deinit(); + } + + pub fn call(self: *Self, a: zml.Buffer, b: zml.Buffer) !void { + if (self.platform.target == .cuda) { + cuda.memcpyToHostAsync(self.a.asHostBuffer().mutBytes(), a.opaqueDeviceMemoryDataPointer(), self.stream); + cuda.memcpyToHostAsync(self.b.asHostBuffer().mutBytes(), b.opaqueDeviceMemoryDataPointer(), self.stream); + + _ = cuda.cuLaunchHostFunc(self.stream, @ptrCast(&add_op_host_func), @ptrCast(self)); + + cuda.memcpyToDeviceAsync(self.results[0].opaqueDeviceMemoryDataPointer(), self.result.asHostBuffer().bytes(), self.stream); + return; + } + + if (self.platform.target == .cpu) { + const result = std.mem.bytesAsValue(f32, self.results[0].asHostBuffer().mutBytes()); + result.* = a.asHostBuffer().item(f32, 0) + b.asHostBuffer().item(f32, 0); + return; + } + + return error.UnsupportedTarget; + } +}; + +const Layer = struct { + pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + const result = zml.custom_call(AddOp, .{ a.print(), b.print() }, &[_]zml.Shape{a.shape()}, .{}); + return result[0]; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const shape = zml.Shape.init(.{}, .f32); + + const buffers: zml.aio.BufferStore.Buffers = .{}; + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); + + const compiled = try compilation.awaitt(); + + var executable = compiled.prepare(model_weights); + defer executable.deinit(); + + executable = try executable.withExecutionContext(); + + try platform.registerFFIType(AddOp); + var add_op: AddOp = try .init(allocator, platform, shape); + defer add_op.deinit(); + try executable.bind(AddOp, &add_op); + + var input_a = [1]f32{1.0}; + var input_buffer_a = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_a), .{}); + defer input_buffer_a.deinit(); + + var input_b = [1]f32{1.0}; + var input_buffer_b = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_b), .{}); + defer input_buffer_b.deinit(); + + var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + defer result.deinit(); + + const cpu_result = try result.toHostAlloc(arena); + _ = cpu_result; // autofix + + // log.warn( + // "\nThe result of {} + {} = {}\n", + // .{ &input_a, &input_b, cpu_result.items(f32) }, + // ); +} diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index d2517a04..b3ce2a74 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -169,7 +169,7 @@ pub const ExecutionContext = opaque { const type_id: TypeId = .{ .type_id = T.type_id }; var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ .ctx = @constCast(self.inner()), - .type_id = @constCast(&type_id.toCStruct()), + .type_id = @constCast(&type_id), }); const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 6e358562..aead3bd3 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1293,7 +1293,7 @@ pub const FFI = extern struct { } }; - pub const RegisterFfiOptions = struct { + pub const RegisterOptions = struct { traits: RegisterHandlerTraits = @enumFromInt(0), }; @@ -1305,7 +1305,7 @@ pub const FFI = extern struct { target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, - options: RegisterFfiOptions, + options: RegisterOptions, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .target_name = target_name.ptr, @@ -1357,7 +1357,3 @@ pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { command_buffer_compatible = c.PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE, _, }; - -pub const CustomCallRegistry = extern struct { - inner: *const c.PJRT_FFI_Register_Handler, -}; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index aaa44f04..88876478 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -31,6 +31,7 @@ zig_library( "aio/torch/py.zig", "buffer.zig", "context.zig", + "custom_call.zig", "dtype.zig", "exe.zig", "floats.zig", diff --git a/zml/buffer.zig b/zml/buffer.zig index 3553bb94..259cb262 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -49,7 +49,7 @@ pub const Buffer = struct { pub const FromOptions = struct { wait: bool = true, - memory: ?pjrt.Memory.Kind = null, + memory: ?Memory = null, }; /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -89,13 +89,13 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); - const memory = for (memories) |m| { + if (opts.memory) |memory| { + const device_memories = try devices[i].addressableMemories(platform.pjrt_api); + const selected_memory = for (device_memories) |m| { const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; + if (kind == memory.toPjrtMemory()) break m; } else return error.NotFound; - args.memory = memory; + args.memory = selected_memory; } else { args.device = devices[i]; } @@ -179,10 +179,9 @@ pub const Buffer = struct { return try from(platform, host_buffer, opts); } - pub fn asPinnedHostBuffer(self: Buffer) HostBuffer { - // TODO restore assert - // const memory = self.getMemory().kind(self._api); - // stdx.debug.assert(memory == .pinned_host, "asPinnedHostBuffer({}) expects a buffer allocated on host memory, got {}. see `toMemory`", .{ self, memory }); + pub fn asHostBuffer(self: Buffer) HostBuffer { + const memory = self.getMemory().kind(self._api); + stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({any}) expects a buffer allocated on host memory, got {any}. see `copyToMemory`", .{ self, memory }); const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } @@ -299,6 +298,11 @@ pub const Buffer = struct { }; } + pub fn opaqueDeviceMemoryDataPointer(self: Buffer) *anyopaque { + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); + return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + } + /// Fetches the content of the given buffer into a stack variable of the given type. pub fn getValue(self: Buffer, T: type) !T { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); @@ -390,13 +394,31 @@ pub const Buffer = struct { return @reduce(.Or, self._shape._sharding_info); } - pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { + pub const CopyToMemoryOpts = struct { + wait: bool = true, + }; + + pub fn copyToMemory(self: Buffer, platform: Platform, memory: Memory, opts: CopyToMemoryOpts) !Buffer { + const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory.toPjrtMemory()); + if (pjrt_memory == null) { + stdx.debug.panic("Memory destination `{s}` for {any}", .{ memory.pjrtName(), self }); + } + var new_shards: Buffer.Shards = .{}; for (self._shards.slice()) |shard| { - const new_shard = try shard.copyToMemory(self._api, memory); + const new_shard = try shard.copyToMemory(self._api, pjrt_memory.?); new_shards.appendAssumeCapacity(new_shard); } + if (opts.wait) { + for (new_shards.constSlice()) |shard| { + const event = shard.getReadyEvent(self._api); + if (event) |e| { + try e.awaitBlocking(self._api); + } + } + } + return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; } diff --git a/zml/context.zig b/zml/context.zig index 15acb93b..f2e01f40 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -13,6 +13,7 @@ const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Target = @import("platform.zig").Target; +const custom_call = @import("custom_call.zig"); const zml_platform = @import("platform.zig"); const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); @@ -36,6 +37,7 @@ pub const Context = struct { inline for (comptime std.enums.values(runtimes.Platform)) |t| { if (runtimes.load(t)) |api| { Context.apis.set(t, api); + if (t == .cuda) cuda.init(); } else |_| {} } } @@ -176,9 +178,9 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); + try custom_call.registerInternalCustomCalls(p); + return p; } @@ -211,71 +213,8 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; }; -const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { - const ffi = platform.pjrt_api.ffi() orelse { - log.warn("Registering custom calls failed: No FFI Extension found in {s} PJRT Plugin.", .{@tagName(platform.target)}); - return; - }; - try ffi.register(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback, .{}); - } - - fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; - std.debug.assert(callback_attr.dtype == .u64); - const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); - - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; - std.debug.assert(user_ctx_ptr.dtype == .u64); - const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.args.buffers()[i]); - } - - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.results.buffers()[i]); - } - - callback(user_ctx, input_buffers, output_buffers); - return null; - } -}; - -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { - // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { - .invalid => @panic("invalid ffi"), - .pred => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), - }; - return Shape.init(buffer_desc.dims(), dt); -} - -/// Create a HostBuffer from a ffi description of a buffer. -/// Normally the ffi describe device buffer but we assume they are located in pinned memory, -/// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { - const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( - buffer_shape, - buffer_desc.data[0..buffer_shape.byteSize()], - ); -} - pub const cuda = struct { pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); pub var cuLaunchHostFunc: CuLaunchHostFunc = @ptrFromInt(0xdeadc00da00); diff --git a/zml/custom_call.zig b/zml/custom_call.zig new file mode 100644 index 00000000..b6cc8f0e --- /dev/null +++ b/zml/custom_call.zig @@ -0,0 +1,224 @@ +const std = @import("std"); + +const asynk = @import("async"); +const stdx = @import("stdx"); + +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; +const mlir = @import("mlir"); +const mlirx = @import("mlirx.zig"); +const pjrt = @import("pjrt"); +const pjrtx = @import("pjrtx.zig"); +const ffi = pjrtx.ffi; +const cuda = @import("context.zig").cuda; +const Buffer = @import("buffer.zig").Buffer; +const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const CompilationContext = @import("module.zig").CompilationContext; +const DataType = @import("dtype.zig").DataType; +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/custom_call"); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn CustomCallInputsType(custom_op: type) type { + const ArgsT = stdx.meta.FnArgs(custom_op.call); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + + const args = @typeInfo(ArgsT).@"struct".fields; + + for (args[1..args.len]) |field| { + if (field.type != Buffer) { + @compileError("Expected all arguments in custom call `call` to be of type zml.Buffer"); + } + } + + return [args.len - 1]Tensor; +} + +pub const CustomCallOptions = struct { + output_operand_aliases: []const i64 = &.{}, + copy_inputs_to_host_pinned: bool = false, + register_ffi_options: pjrt.FFI.RegisterOptions = .{ + .traits = @enumFromInt(0), + }, +}; + +pub fn custom_call( + comptime custom_op: type, + inputs: CustomCallInputsType(custom_op), + res_shapes_: []const Shape, + comptime opts: CustomCallOptions, +) []Tensor { + stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); + const op_name = @typeName(custom_op); + + const ctx = CompilationContext.current(); + const allocator = ctx.allocator(); + const mlir_ctx = ctx.mlirCtx(); + const platform = ctx._platform; + const pjrt_api = platform.pjrt_api; + const ffi_ = pjrt_api.ffi(); + + if (ffi_ == null) { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); + } + + const ffi_func = proxy(custom_op, opts); + const target_name = "callback_" ++ op_name; + + ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.register_ffi_options) catch unreachable; + log.info("{s} / Registered custom call with target name \"{s}\" with proxy func {*}", .{ op_name, target_name, &ffi_func }); + + const custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; + for (inputs, 0..) |t, i| { + custom_call_inputs[i] = t.value(); + } + + const res_types = allocator.alloc(mlir.Type, 8) catch unreachable; + for (res_shapes_, 0..) |sh, i| { + res_types[i] = mlirx.tensorType(mlir_ctx, sh); + } + + const frontend_attributes: mlir.Attribute = .dict(mlir_ctx, &.{}); + + const op = dialect.stablehlo.custom_call( + mlir_ctx, + custom_call_inputs[0..inputs.len], + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, + .has_side_effect = true, + .output_operand_aliases = opts.output_operand_aliases, + }, + res_types[0..res_shapes_.len], + mlir_ctx.location(@src()), + ); + + var custom_call_outputs = allocator.alloc(Tensor, 8) catch unreachable; + + for (custom_call_outputs[0..res_shapes_.len], res_shapes_, 0..) |*t, sh, i| { + t.* = Tensor._result(sh, op.result(i)); + } + + return custom_call_outputs[0..res_shapes_.len]; +} + +fn proxy(comptime custom_op: type, opts: CustomCallOptions) ffi.Handler { + return struct { + const Self = @This(); + + pub fn proxy(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { + if (call_frame.registeringHook()) return null; + const execution_context = call_frame.ctx; + const user_ctx: *custom_op = ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api) catch unreachable; + const platform: Platform = user_ctx.platform; + + if (@hasField(custom_op, "stream") and platform.target != .cpu) { + const stream = call_frame.api.stream(execution_context); + user_ctx.stream = stream; + } else { + log.info("No stream field provided in container {s} for custom call", .{@typeName(custom_op)}); + } + + var callback_args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; + callback_args[0] = user_ctx; + + inline for (1..callback_args.len) |i| { + const ffi_buffer = call_frame.args.buffers()[i - 1]; + const ffi_buffer_shape = getShape(ffi_buffer); + var zml_buffer: Buffer = undefined; + + if (platform.target == .cpu) { + zml_buffer = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + zml_buffer = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch unreachable; + } + callback_args[i] = zml_buffer; + } + + for (0..call_frame.results.len) |i| { + const ffi_buffer = call_frame.results.buffers()[i]; + const ffi_buffer_shape = getShape(ffi_buffer); + + if (platform.target == .cpu) { + user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + } + + @call(.auto, custom_op.call, callback_args) catch |err| { + stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(custom_op), err }); + }; + + return null; + } + }.proxy; +} + +pub fn getShape(ffi_buffer: *const ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .pred => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} + +/// Internal custom calls. +/// These are not meant to be used by users, but rather by the library itself. +pub const custom_call_internal_types = [_]type{ + Print, +}; + +pub fn registerInternalCustomCalls( + platform: Platform, +) !void { + inline for (custom_call_internal_types) |custom_call_type| { + try platform.registerFFIType(custom_call_type); + log.info("Registered internal custom call {s} with type_id {any}", .{ @typeName(custom_call_type), custom_call_type.type_id }); + } +} + +pub const Print = struct { + pub var type_id: i64 = undefined; + const Self = @This(); + + allocator: std.mem.Allocator, + platform: Platform, + + results: [1]Buffer = undefined, + + pub fn init( + allocator: std.mem.Allocator, + platform: Platform, + ) !Print { + return .{ + .allocator = allocator, + .platform = platform, + }; + } + + pub fn call(_: *Self, input: Buffer) !void { + std.log.defaultLog(.info, .zml, "Device buffer: {any}: {any}", .{ input.shape(), input.asHostBuffer().pretty() }); + } +}; diff --git a/zml/exe.zig b/zml/exe.zig index 625fecf9..7a303fa3 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -8,6 +8,7 @@ const Bufferized = @import("tensor.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); +const custom_call = @import("custom_call.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; @@ -285,6 +286,21 @@ pub const BaseExe = struct { stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); } + pub fn bind(self: BaseExe, comptime T: type, value: *T) !void { + stdx.debug.assert(self.context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = self.platform.pjrt_api; + + if (pjrt_api.ffi()) |ffi| { + const type_id = T.type_id; + const user_data: *anyopaque = @ptrCast(@constCast(value)); + + try ffi.addUserData(pjrt_api, self.context.?, .{ .type_id = type_id, .user_data = user_data }); + log.info("Bound {s}@{any} with type id {any} on {any}", .{ @typeName(T), user_data, T.type_id, self.context.? }); + } else { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(self.platform.target)}); + } + } + pub fn serialize(self: BaseExe, writer: anytype) !void { var executable = try self.exe.getExecutable(self.platform.pjrt_api); var serialize_result = try executable.serialize(self.platform.pjrt_api); @@ -348,6 +364,28 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + pub fn withExecutionContext(self: Self) !Self { + stdx.debug.assert(self.inner.context == null, "Exe already has an execution context", .{}); + var new: Self = .{ .inner = self.inner }; + var arena = &new.inner._arena; + var allocator = arena.allocator(); + + const pjrt_execute_context = try new.inner.platform.pjrt_api.createExecuteContext(); + new.inner.context = pjrt_execute_context; + inline for (custom_call.custom_call_internal_types) |custom_call_internal_type| { + const value_ptr = try allocator.create(custom_call_internal_type); + value_ptr.* = try .init(allocator, new.platform()); + try new.inner.bind(custom_call_internal_type, value_ptr); + } + log.info("Created context execution {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); + + return new; + } + + pub fn bind(self: Self, comptime T: type, value: *T) !void { + try self.inner.bind(T, value); + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index c2099b0d..ee598880 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -176,10 +176,20 @@ pub const HostBuffer = struct { return ptr[0..self._shape.count()]; } + pub fn item(self: HostBuffer, comptime T: type, index: usize) T { + stdx.debug.assert(index < self._shape.count(), "index {any} out of bounds for shape {any}", .{ index, self._shape }); + return self.items(T)[index]; + } + pub fn mutItems(self: HostBuffer, comptime T: type) []T { return @constCast(self.items(T)); } + pub fn mutitem(self: HostBuffer, comptime T: type, index: usize) T { + stdx.debug.assert(index < self._shape.count(), "index {any} out of bounds for shape {any}", .{ index, self._shape }); + return self.mutItems(T)[index]; + } + pub fn bytes(self: HostBuffer) []const u8 { stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); return self._data[0..self._shape.byteSize()]; diff --git a/zml/module.zig b/zml/module.zig index 33d22f78..6263d3fd 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -1178,9 +1178,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1196,7 +1196,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/ops.zig b/zml/ops.zig index c26eac3c..ee125086 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -764,33 +764,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - return customCall( - "zmlHostBufferCallback", - inputs, - output_shapes, - .{ - .callback = @intFromPtr(callback), - .user_context = @intFromPtr(blkctx), - }, - .{ - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = opts.output_operand_aliases, - }, - ); -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 84343a63..9239bc3c 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -207,6 +207,13 @@ pub const Event = opaque { return self.inner().getEventError(api); } + pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { + if (self.isReady(api)) { + return; + } + try self.inner().await_(api); + } + pub fn await_(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); diff --git a/zml/tensor.zig b/zml/tensor.zig index 26f670d7..04874214 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -13,6 +13,8 @@ const Memory = @import("buffer.zig").Buffer.Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); +const custom_call_ = @import("custom_call.zig"); +const custom_call = custom_call_.custom_call; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -3824,22 +3826,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - // TODO: find a way of doing print that doesn't involve a H2D copy. - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.log.defaultLog(.info, .zml, "Device buffer: {f}: {f}", .{ host_buffer.shape(), host_buffer.pretty() }); - // This is true because of the operand aliases. - // Since the result is already pointing to the input we don't need to modify the buffer. - std.debug.assert(host_buffer._data == outputs[0]._data); + return custom_call(custom_call_.Print, .{input}, &[_]Shape{input.shape()}, .{ .output_operand_aliases = &.{0}, .copy_inputs_to_host_pinned = true })[0]; } }; diff --git a/zml/zml.zig b/zml/zml.zig index b5cd8039..05602e9a 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -17,6 +17,7 @@ pub const Context = @import("context.zig").Context; pub const Data = @import("dtype.zig").Data; pub const DataType = @import("dtype.zig").DataType; pub const exe = @import("exe.zig"); +pub const custom_call = @import("custom_call.zig").custom_call; pub const compile = exe.compile; pub const compileWithPrefix = exe.compileWithPrefix; pub const compileFn = exe.compileFn; From 3e49bfee282d6e912e159af936f6155cecb17256 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 11 Jun 2025 15:47:50 +0000 Subject: [PATCH 2/4] async exe --- zml/context.zig | 2 -- zml/custom_call.zig | 20 ++++++++++---------- zml/exe.zig | 12 ++++++------ zml/pjrtx.zig | 12 ++++++++++-- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/zml/context.zig b/zml/context.zig index f2e01f40..47c86708 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -7,8 +7,6 @@ const runfiles = @import("runfiles"); const runtimes = @import("runtimes"); const stdx = @import("stdx"); -const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; diff --git a/zml/custom_call.zig b/zml/custom_call.zig index b6cc8f0e..849a854e 100644 --- a/zml/custom_call.zig +++ b/zml/custom_call.zig @@ -1,25 +1,25 @@ const std = @import("std"); const asynk = @import("async"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); const stdx = @import("stdx"); -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; -}; -const mlir = @import("mlir"); +const Buffer = @import("buffer.zig").Buffer; +const CompilationContext = @import("module.zig").CompilationContext; +const cuda = @import("context.zig").cuda; +const DataType = @import("dtype.zig").DataType; +const HostBuffer = @import("hostbuffer.zig").HostBuffer; const mlirx = @import("mlirx.zig"); -const pjrt = @import("pjrt"); const pjrtx = @import("pjrtx.zig"); const ffi = pjrtx.ffi; -const cuda = @import("context.zig").cuda; -const Buffer = @import("buffer.zig").Buffer; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; -const CompilationContext = @import("module.zig").CompilationContext; -const DataType = @import("dtype.zig").DataType; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; const log = std.log.scoped(.@"zml/custom_call"); pub const std_options: std.Options = .{ diff --git a/zml/exe.zig b/zml/exe.zig index 7a303fa3..37fff489 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -250,11 +250,11 @@ pub const BaseExe = struct { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn _unsafeAssignResults(self: BaseExe, T: type, result: *T) void { @@ -330,7 +330,7 @@ pub const BaseExe = struct { pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ - .n_in = self.input_buffer_count, + .input_shapes = self.input_shapes, .result_shapes = self.result_shapes, .n_devices = self.num_devices, }); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 9239bc3c..b8ef9b3f 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -271,14 +271,22 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, .context = args.context, - } }); + }); + // try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + // .num_args = args.num_args, + // .arguments = @ptrCast(args.arguments), + // .results = @ptrCast(args.results), + // .events = @ptrCast(args.events), + // .non_donatable_input_indices = args.non_donatable_input_indices, + // .context = args.context, + // } }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 0ed921f49c766d619753d95b9528136fd94f22dc Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 15 Jul 2025 09:58:40 +0200 Subject: [PATCH 3/4] handle memories on CPU --- zml/buffer.zig | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 259cb262..1da9a094 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -89,15 +89,20 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (opts.memory) |memory| { + if (platform.target == .cpu or opts.memory == null) { + args.device = devices[i]; + } else { + const memory = opts.memory.?; const device_memories = try devices[i].addressableMemories(platform.pjrt_api); + // TODO measure the cost of this and consider caching on Zig side inside the platform. const selected_memory = for (device_memories) |m| { const kind = m.kind(platform.pjrt_api); if (kind == memory.toPjrtMemory()) break m; - } else return error.NotFound; + } else { + log.warn("Platform {s} doesn't have memory {s}", .{ @tagName(platform.target), @tagName(memory) }); + return error.NotFound; + }; args.memory = selected_memory; - } else { - args.device = devices[i]; } const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); From 64ed5fefc99a8992a33875fb99e9dbc5ff6c0cef Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 3 Sep 2025 16:07:40 +0000 Subject: [PATCH 4/4] fixes post zig 0.15 bump --- examples/custom_call/main.zig | 9 ++++----- zml/buffer.zig | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig index e5a17443..2868dd18 100644 --- a/examples/custom_call/main.zig +++ b/examples/custom_call/main.zig @@ -145,10 +145,9 @@ pub fn asyncMain() !void { defer result.deinit(); const cpu_result = try result.toHostAlloc(arena); - _ = cpu_result; // autofix - // log.warn( - // "\nThe result of {} + {} = {}\n", - // .{ &input_a, &input_b, cpu_result.items(f32) }, - // ); + log.warn( + "\nThe result of {d} + {d} = {d}\n", + .{ input_a[0], input_b[0], cpu_result.items(f32)[0] }, + ); } diff --git a/zml/buffer.zig b/zml/buffer.zig index 1da9a094..c7626f7d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -185,8 +185,8 @@ pub const Buffer = struct { } pub fn asHostBuffer(self: Buffer) HostBuffer { - const memory = self.getMemory().kind(self._api); - stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({any}) expects a buffer allocated on host memory, got {any}. see `copyToMemory`", .{ self, memory }); + // const memory = self.getMemory().kind(self._api); + // stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({any}) expects a buffer allocated on host memory, got {any}. see `copyToMemory`", .{ self, memory }); const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); }