From 2be964ba9abd06d5b860c646c8a607d3dd16a85a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 20 Jun 2025 12:35:08 +0000 Subject: [PATCH 01/17] zml/exe: don't wait for event Don't forget to delete them --- zml/exe.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/exe.zig b/zml/exe.zig index 0923f17c..d5c26795 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -251,7 +251,7 @@ pub const BaseExe = struct { for (events[0..sharding.num_partitions]) |e| { if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; + ev.deinit(self.platform.pjrt_api); } } } From 637e67a07bf397d417e93cc9affc1af1899d3249 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 20 Jun 2025 12:35:31 +0000 Subject: [PATCH 02/17] zml/pjrtx: remove async blocking call to execute --- zml/pjrtx.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 1795efc3..9e3143d6 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -272,14 +272,14 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, .context = args.context, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 497e4b8d3d9eb5172ce4e950bcd88cdd8233a083 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 26 Jun 2025 12:37:28 +0000 Subject: [PATCH 03/17] zml/module.zig: disable gemm autotuning on ROCm This is taking way too much time for llmd --- zml/module.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/zml/module.zig b/zml/module.zig index 8ab1aedc..dcb647fc 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -901,6 +901,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m // Use lld from libllvm instead of invoking the ld.lld binary. // This saves us from having to sandbox it. setFlag(&options, "xla_gpu_use_inprocess_lld", true); + setFlag(&options, "xla_gpu_autotune_level", 0); }, else => {}, } From 8c889d48a39486c404a3013c57c6fa04134ea1b4 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 26 Jun 2025 12:55:39 +0000 Subject: [PATCH 04/17] zml/tools/tracer.zig: make FakeTracer api compatible with CudaTracer --- zml/tools/tracer.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 06bcd6fa..7efe748a 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -1,4 +1,5 @@ const builtin = @import("builtin"); + const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { @@ -83,6 +84,10 @@ const FakeTracer = struct { return .{}; } + pub fn deinit(self: *const FakeTracer) void { + _ = self; + } + pub fn event(self: *const FakeTracer, message: [:0]const u8) void { _ = self; _ = message; From b39b92829c1621e676504b94f5e7be43ef36114a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 9 Jul 2025 11:34:29 +0000 Subject: [PATCH 05/17] stdx/math: added cdiv function --- stdx/math.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stdx/math.zig b/stdx/math.zig index da0bec79..1e1e1318 100644 --- a/stdx/math.zig +++ b/stdx/math.zig @@ -21,3 +21,7 @@ pub inline fn roundeven(x: anytype) @TypeOf(x) { extern fn @"llvm.roundeven"(@TypeOf(x)) @TypeOf(x); }.@"llvm.roundeven"(x); } + +pub fn cdiv(numerator: anytype, denominator: anytype) @TypeOf(numerator) { + return (numerator + denominator - 1) / denominator; +} From 68e9995373b45b96f8f7fae1d8faf95c9a56aa9d Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 11 Jul 2025 15:36:19 +0000 Subject: [PATCH 06/17] zml/nn: correctly parse RopeOpts.Scaling from JSON --- zml/nn.zig | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index ffdb7a9c..adc06237 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -175,17 +175,23 @@ pub const RopeOpts = struct { /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { - const content = try std.json.Value.jsonParse(allocator, source, options); - if (content == .null) return .default; + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } - if (content != .object) return error.InvalidEnumTag; + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { + if (source == .null) return .default; - const obj = content.object; + if (source != .object) return error.InvalidEnumTag; + + const obj = source.object; const impl = obj.get("rope_type") orelse return error.MissingField; if (impl != .string) return error.InvalidEnumTag; if (std.mem.eql(u8, impl.string, "llama3")) { - // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, undefined, content, .{ .ignore_unknown_fields = true }) }; + return .{ .llama3 = try std.json.innerParseFromValue(Llama3, allocator, source, options) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; From baca71ec3b2a5582461608e80010d0770ac5524a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 16 Jul 2025 13:51:24 +0000 Subject: [PATCH 07/17] zml/tools: temporarily disable CudaTracer for universal binaries --- zml/tools/tracer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 7efe748a..3d0c1b31 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -4,7 +4,7 @@ const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { .macos => MacOsTracer, - .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + .linux => FakeTracer, else => FakeTracer, }; From 8c07983c047bc19a6225226555a74d1889b25432 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 21 Jul 2025 18:41:16 +0200 Subject: [PATCH 08/17] coro without types --- async/async.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ async/coro.zig | 21 +++++++++++++++++++-- zml/tools/tracer.zig | 4 ++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/async/async.zig b/async/async.zig index f14608b8..a58b9b13 100644 --- a/async/async.zig +++ b/async/async.zig @@ -10,6 +10,8 @@ const coro = @import("coro.zig"); const executor = @import("executor.zig"); const stack = @import("stack.zig"); +pub const Coro = coro.Coro; + test { std.testing.refAllDecls(@This()); std.testing.refAllDecls(coro); @@ -65,6 +67,46 @@ fn FrameExx(comptime func: anytype, comptime argsT: type, comptime returnT: type }; } +/// Create a Coro +/// self and stack pointers must remain stable for the lifetime of +/// the coroutine. +pub fn async2(comptime func: anytype, args: anytype, result_dst: anytype) !*coro.Coro { + const Sig = stdx.meta.FnSignature(func, @TypeOf(args)); + var new_stack = try AsyncThread.current.stack_allocator.create(); + const result_dst_storage = try new_stack.push(*Sig.ReturnT); + result_dst_storage.* = @alignCast(@ptrCast(result_dst)); + const args_storage = try new_stack.push(Sig.ArgsT); + args_storage.* = args; + + const Helpers = Helpers_async2(Sig); + const frame = try Coro.initFromStack(Helpers.wrapfn, new_stack, args_storage); + frame.switchIn(); + return frame; +} + +pub fn awaitAll(futures: []const *coro.Coro) void { + for (futures) |fut| fut.@"await"(); +} + +fn Helpers_async2(comptime Sig: stdx.meta.Signature) type { + return struct { + pub fn wrapfn() void { + const storage = Coro.current().storage; + const res_dst_storage: *(*Sig.ReturnT) = @alignCast(@ptrCast(storage)); + const res_dst = res_dst_storage.*; + + const offset = @max(@sizeOf(usize), @alignOf(Sig.ArgsT)); + const res_args_storage: *const Sig.ArgsT = @ptrFromInt(@intFromPtr(storage) + offset); + + res_dst.* = @call( + .auto, + Sig.Func.Value, + res_args_storage.*, + ); + } + }; +} + pub fn asyncc(comptime func: anytype, args: anytype) !FrameEx(func, @TypeOf(args)) { const Signature = stdx.meta.FnSignature(func, @TypeOf(args)); const new_stack = try AsyncThread.current.stack_allocator.create(); diff --git a/async/coro.zig b/async/coro.zig index be26b95b..95eefac2 100644 --- a/async/coro.zig +++ b/async/coro.zig @@ -104,7 +104,7 @@ pub fn xsuspendSafe() Error!void { thread_state.switchOut(coro.resumer); } -const Coro = struct { +pub const Coro = struct { /// Coroutine status const Status = enum { Start, @@ -137,7 +137,7 @@ const Coro = struct { // empty } - fn initFromStack(func: *const fn () void, stack_: stack.Stack, storage: ?*anyopaque) !Frame { + pub fn initFromStack(func: *const fn () void, stack_: stack.Stack, storage: ?*anyopaque) !Frame { // try StackOverflow.setMagicNumber(stack.full); var stack__ = stack_; const coro = try stack__.push(Coro); @@ -180,6 +180,19 @@ const Coro = struct { @tagName(self.status), }); } + + pub fn @"await"(self: *Coro) void { + while (self.status != .Done) xsuspend(); + std.debug.assert(self.status == .Done); + } + + pub fn switchIn(self: *Coro) void { + thread_state.switchIn(self); + } + + pub fn current() *Coro { + return thread_state.current_coro.?; + } }; const VoidSignature = CoroT.Signature.init((struct { @@ -260,6 +273,10 @@ const CoroT = struct { Sig.Func.Value, storage.args, ); + switch (@typeInfo(Sig.ReturnT)) { + .error_union => _ = storage.retval catch unreachable, + else => {}, + } } }; } diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 3d0c1b31..597d3ee6 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -61,6 +61,10 @@ const MacOsTracer = struct { }; } + pub fn deinit(self: *const MacOsTracer) void { + _ = self; + } + pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { const interval_id = c.os_signpost_id_generate(self.logger); c.zml_os_signpost_event(self.logger, interval_id, message); From a5946429d70c38e9ce5536d7d42ac9546e362727 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Jul 2025 11:10:02 +0200 Subject: [PATCH 09/17] async2 --- async/async.zig | 22 +++++++++------------- async/coro.zig | 5 ++++- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/async/async.zig b/async/async.zig index a58b9b13..72503c22 100644 --- a/async/async.zig +++ b/async/async.zig @@ -73,13 +73,12 @@ fn FrameExx(comptime func: anytype, comptime argsT: type, comptime returnT: type pub fn async2(comptime func: anytype, args: anytype, result_dst: anytype) !*coro.Coro { const Sig = stdx.meta.FnSignature(func, @TypeOf(args)); var new_stack = try AsyncThread.current.stack_allocator.create(); - const result_dst_storage = try new_stack.push(*Sig.ReturnT); - result_dst_storage.* = @alignCast(@ptrCast(result_dst)); - const args_storage = try new_stack.push(Sig.ArgsT); - args_storage.* = args; - + // Note: I'm not a big fan of this struct. It would be simpler if res_dst was part of the Coro itself. + const storage = try new_stack.push(struct { res_dst: *anyopaque, args: Sig.ArgsT }); + storage.args = args; + storage.res_dst = result_dst; const Helpers = Helpers_async2(Sig); - const frame = try Coro.initFromStack(Helpers.wrapfn, new_stack, args_storage); + const frame = try Coro.initFromStack(Helpers.wrapfn, new_stack, storage); frame.switchIn(); return frame; } @@ -91,17 +90,14 @@ pub fn awaitAll(futures: []const *coro.Coro) void { fn Helpers_async2(comptime Sig: stdx.meta.Signature) type { return struct { pub fn wrapfn() void { - const storage = Coro.current().storage; - const res_dst_storage: *(*Sig.ReturnT) = @alignCast(@ptrCast(storage)); - const res_dst = res_dst_storage.*; - - const offset = @max(@sizeOf(usize), @alignOf(Sig.ArgsT)); - const res_args_storage: *const Sig.ArgsT = @ptrFromInt(@intFromPtr(storage) + offset); + const storage_raw = Coro.current().storage; + const storage: *struct { res_dst: *anyopaque, args: Sig.ArgsT } = @alignCast(@ptrCast(storage_raw)); + const res_dst: *Sig.ReturnT = @alignCast(@ptrCast(storage.res_dst)); res_dst.* = @call( .auto, Sig.Func.Value, - res_args_storage.*, + storage.args, ); } }; diff --git a/async/coro.zig b/async/coro.zig index 95eefac2..e4172ed6 100644 --- a/async/coro.zig +++ b/async/coro.zig @@ -175,9 +175,12 @@ pub const Coro = struct { pub fn format(self: Coro, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { _ = fmt; _ = options; - try writer.print("Coro{{.id = {any}, .status = {s}}}", .{ + try writer.print("Coro{{.id = {any}, .status = {s}, .storage={x}, .stack_start={x}, .stack_used={x}}}", .{ self.id, @tagName(self.status), + @intFromPtr(self.storage), + @intFromPtr(self.stack.full), + @intFromPtr(self.stack.used.ptr), }); } From 46d897ca9df7e571235519fd343ca60e24fd4bb7 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Jul 2025 11:10:19 +0200 Subject: [PATCH 10/17] fallback for MemoryStats --- pjrt/pjrt.zig | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 58d9ac29..bdb017d7 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -578,9 +578,12 @@ pub const Device = opaque { } pub fn memoryStats(self: *const Device, api: *const Api) ApiError!MemoryStats { - const ret = try api.call(.PJRT_Device_MemoryStats, .{ + const ret = api.call(.PJRT_Device_MemoryStats, .{ .device = self.inner(), - }); + }) catch |err| switch (err) { + error.Unimplemented => return std.mem.zeroes(MemoryStats), + else => return err, + }; return .{ .bytes_in_use = @intCast(ret.bytes_in_use), .peak_bytes_in_use = @intCast(ret.peak_bytes_in_use), From 28a482d4469b67e82e7e393a727929fc2eb50569 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 27 May 2025 17:20:58 +0200 Subject: [PATCH 11/17] zml: custom call v2 --- examples/custom_call/BUILD | 10 ++ examples/custom_call/main.zig | 153 +++++++++++++++++++++++ examples/llama/llama.zig | 1 + pjrt/ffi.zig | 2 +- pjrt/pjrt.zig | 8 +- zml/buffer.zig | 46 +++++-- zml/context.zig | 7 +- zml/custom_call.zig | 224 ++++++++++++++++++++++++++++++++++ zml/exe.zig | 38 ++++++ zml/hostbuffer.zig | 10 ++ zml/module.zig | 8 +- zml/ops.zig | 27 ---- zml/pjrtx.zig | 7 ++ zml/tensor.zig | 19 +-- zml/zml.zig | 1 + 15 files changed, 491 insertions(+), 70 deletions(-) create mode 100644 examples/custom_call/BUILD create mode 100644 examples/custom_call/main.zig create mode 100644 zml/custom_call.zig diff --git a/examples/custom_call/BUILD b/examples/custom_call/BUILD new file mode 100644 index 00000000..bc605442 --- /dev/null +++ b/examples/custom_call/BUILD @@ -0,0 +1,10 @@ +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "custom_call", + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig new file mode 100644 index 00000000..e8bdefd6 --- /dev/null +++ b/examples/custom_call/main.zig @@ -0,0 +1,153 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +const cuda = zml.context.cuda; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +fn add_op_host_func(data: *anyopaque) callconv(.c) void { + const ctx = @as(*AddOp, @alignCast(@ptrCast(data))); + + const result = std.mem.bytesAsValue(f32, ctx.result.asHostBuffer().mutBytes()); + result.* = ctx.a.asHostBuffer().item(f32, 0) + ctx.b.asHostBuffer().item(f32, 0); +} + +pub const AddOp = struct { + pub var type_id: i64 = undefined; + const Self = @This(); + + allocator: std.mem.Allocator, + platform: zml.Platform, + + a: zml.Buffer, + b: zml.Buffer, + result: zml.Buffer, + + results: [1]zml.Buffer = undefined, + stream: *anyopaque = undefined, + + pub fn init( + allocator: std.mem.Allocator, + platform: zml.Platform, + io_shape: zml.Shape, + ) !AddOp { + const host_buffer = try zml.HostBuffer.empty(allocator, io_shape); + defer host_buffer.deinit(allocator); + + const a = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + const b = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + const result = try zml.Buffer.from(platform, host_buffer, .{ .memory = .host_pinned }); + + return .{ + .a = a, + .b = b, + .result = result, + .platform = platform, + .allocator = allocator, + }; + } + + pub fn deinit(self: *Self) void { + self.a.deinit(); + self.b.deinit(); + self.result.deinit(); + } + + pub fn call(self: *Self, a: zml.Buffer, b: zml.Buffer) !void { + if (self.platform.target == .cuda) { + cuda.memcpyToHostAsync(self.a.asHostBuffer().mutBytes(), a.opaqueDeviceMemoryDataPointer(), self.stream); + cuda.memcpyToHostAsync(self.b.asHostBuffer().mutBytes(), b.opaqueDeviceMemoryDataPointer(), self.stream); + + _ = cuda.cuLaunchHostFunc(self.stream, @ptrCast(&add_op_host_func), @ptrCast(self)); + + cuda.memcpyToDeviceAsync(self.results[0].opaqueDeviceMemoryDataPointer(), self.result.asHostBuffer().bytes(), self.stream); + return; + } + + if (self.platform.target == .cpu) { + const result = std.mem.bytesAsValue(f32, self.results[0].asHostBuffer().mutBytes()); + result.* = a.asHostBuffer().item(f32, 0) + b.asHostBuffer().item(f32, 0); + return; + } + + return error.UnsupportedTarget; + } +}; + +const Layer = struct { + pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + const result = zml.custom_call(AddOp, .{ a.print(), b.print() }, &[_]zml.Shape{a.shape()}, .{}); + return result[0]; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const shape = zml.Shape.init(.{}, .f32); + + const buffers: zml.aio.BufferStore.Buffers = .{}; + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); + + const compiled = try compilation.awaitt(); + + var executable = compiled.prepare(model_weights); + defer executable.deinit(); + + executable = try executable.withExecutionContext(); + + try platform.registerFFIType(AddOp); + var add_op: AddOp = try .init(allocator, platform, shape); + defer add_op.deinit(); + try executable.bind(AddOp, &add_op); + + var input_a = [1]f32{1.0}; + var input_buffer_a = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_a), .{}); + defer input_buffer_a.deinit(); + + var input_b = [1]f32{1.0}; + var input_buffer_b = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_b), .{}); + defer input_buffer_b.deinit(); + + var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + defer result.deinit(); + + var cpu_result = try result.toHostAlloc(arena); + + log.warn( + "\nThe result of {d} + {d} = {d}\n", + .{ &input_a, &input_b, cpu_result.items(f32) }, + ); +} diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index ee9c852f..182672d4 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -46,6 +46,7 @@ pub const LlamaLM = struct { self.model.max_seq_len = @intCast(options.max_seq_len); self.model.num_heads = @intCast(config.num_attention_heads); self.model.num_kv_heads = @intCast(config.num_key_value_heads); + self.model.norm.eps = config.rms_norm_eps; self.model.rope_opts = .{ .layout = if (config.hf_rope_impl) .sequential else .interleaved, .freq_base = config.rope_theta, diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index d4fb6ee4..3d6376ad 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -169,7 +169,7 @@ pub const ExecutionContext = opaque { const type_id: TypeId = .{ .type_id = T.type_id }; var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ .ctx = @constCast(self.inner()), - .type_id = @constCast(&type_id.toCStruct()), + .type_id = @constCast(&type_id), }); const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 58d9ac29..f13de960 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1307,7 +1307,7 @@ pub const FFI = extern struct { } }; - pub const RegisterFfiOptions = struct { + pub const RegisterOptions = struct { traits: RegisterHandlerTraits = @enumFromInt(0), }; @@ -1319,7 +1319,7 @@ pub const FFI = extern struct { target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, - options: RegisterFfiOptions, + options: RegisterOptions, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .api_version = 1, @@ -1372,7 +1372,3 @@ pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { command_buffer_compatible = c.PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE, _, }; - -pub const CustomCallRegistry = extern struct { - inner: *const c.PJRT_FFI_Register_Handler, -}; diff --git a/zml/buffer.zig b/zml/buffer.zig index 36f61255..b6f18fbf 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -49,7 +49,7 @@ pub const Buffer = struct { pub const FromOptions = struct { wait: bool = true, - memory: ?pjrt.Memory.Kind = null, + memory: ?Memory = null, }; /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -89,13 +89,13 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); - const memory = for (memories) |m| { + if (opts.memory) |memory| { + const device_memories = try devices[i].addressableMemories(platform.pjrt_api); + const selected_memory = for (device_memories) |m| { const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; + if (kind == memory.toPjrtMemory()) break m; } else return error.NotFound; - args.memory = memory; + args.memory = selected_memory; } else { args.device = devices[i]; } @@ -179,10 +179,9 @@ pub const Buffer = struct { return try from(platform, host_buffer, opts); } - pub fn asPinnedHostBuffer(self: Buffer) HostBuffer { - // TODO restore assert - // const memory = self.getMemory().kind(self._api); - // stdx.debug.assert(memory == .pinned_host, "asPinnedHostBuffer({}) expects a buffer allocated on host memory, got {}. see `toMemory`", .{ self, memory }); + pub fn asHostBuffer(self: Buffer) HostBuffer { + const memory = self.getMemory().kind(self._api); + stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({}) expects a buffer allocated on host memory, got {}. see `copyToMemory`", .{ self, memory }); const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } @@ -299,6 +298,11 @@ pub const Buffer = struct { }; } + pub fn opaqueDeviceMemoryDataPointer(self: Buffer) *anyopaque { + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); + return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + } + /// Fetches the content of the given buffer into a stack variable of the given type. pub fn getValue(self: Buffer, T: type) !T { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); @@ -394,13 +398,31 @@ pub const Buffer = struct { return @reduce(.Or, self._shape._sharding_info); } - pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { + pub const CopyToMemoryOpts = struct { + wait: bool = true, + }; + + pub fn copyToMemory(self: Buffer, platform: Platform, memory: Memory, opts: CopyToMemoryOpts) !Buffer { + const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory.toPjrtMemory()); + if (pjrt_memory == null) { + stdx.debug.panic("Memory destination `{s}` for {}", .{ memory.pjrtName(), self }); + } + var new_shards: Buffer.Shards = .{}; for (self._shards.slice()) |shard| { - const new_shard = try shard.copyToMemory(self._api, memory); + const new_shard = try shard.copyToMemory(self._api, pjrt_memory.?); new_shards.appendAssumeCapacity(new_shard); } + if (opts.wait) { + for (new_shards.constSlice()) |shard| { + const event = shard.getReadyEvent(self._api); + if (event) |e| { + try e.awaitBlocking(self._api); + } + } + } + return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; } diff --git a/zml/context.zig b/zml/context.zig index bde867f9..028f88b3 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -13,6 +13,7 @@ const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Target = @import("platform.zig").Target; +const custom_call = @import("custom_call.zig"); const zml_platform = @import("platform.zig"); const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); @@ -176,9 +177,9 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); + try custom_call.registerInternalCustomCalls(p); + return p; } @@ -211,8 +212,6 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; }; const CustomCall = struct { diff --git a/zml/custom_call.zig b/zml/custom_call.zig new file mode 100644 index 00000000..1c7a2f49 --- /dev/null +++ b/zml/custom_call.zig @@ -0,0 +1,224 @@ +const std = @import("std"); + +const asynk = @import("async"); +const stdx = @import("stdx"); + +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; +const mlir = @import("mlir"); +const mlirx = @import("mlirx.zig"); +const pjrt = @import("pjrt"); +const pjrtx = @import("pjrtx.zig"); +const ffi = pjrtx.ffi; +const cuda = @import("context.zig").cuda; +const Buffer = @import("buffer.zig").Buffer; +const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const CompilationContext = @import("module.zig").CompilationContext; +const DataType = @import("dtype.zig").DataType; +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/custom_call"); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn CustomCallInputsType(custom_op: type) type { + const ArgsT = stdx.meta.FnArgs(custom_op.call); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + + const args = @typeInfo(ArgsT).@"struct".fields; + + for (args[1..args.len]) |field| { + if (field.type != Buffer) { + @compileError("Expected all arguments in custom call `call` to be of type zml.Buffer"); + } + } + + return [args.len - 1]Tensor; +} + +pub const CustomCallOptions = struct { + output_operand_aliases: []const i64 = &.{}, + copy_inputs_to_host_pinned: bool = false, + register_ffi_options: pjrt.FFI.RegisterOptions = .{ + .traits = @enumFromInt(0), + }, +}; + +pub fn custom_call( + comptime custom_op: type, + inputs: CustomCallInputsType(custom_op), + res_shapes_: []const Shape, + comptime opts: CustomCallOptions, +) []Tensor { + stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); + const op_name = @typeName(custom_op); + + const ctx = CompilationContext.current(); + const allocator = ctx.allocator(); + const mlir_ctx = ctx.mlirCtx(); + const platform = ctx._platform; + const pjrt_api = platform.pjrt_api; + const ffi_ = pjrt_api.ffi(); + + if (ffi_ == null) { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); + } + + const ffi_func = proxy(custom_op, opts); + const target_name = "callback_" ++ op_name; + + ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.register_ffi_options) catch unreachable; + log.info("{s} / Registered custom call with target name \"{s}\" with proxy func {*}", .{ op_name, target_name, &ffi_func }); + + const custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; + for (inputs, 0..) |t, i| { + custom_call_inputs[i] = t.value(); + } + + const res_types = allocator.alloc(mlir.Type, 8) catch unreachable; + for (res_shapes_, 0..) |sh, i| { + res_types[i] = mlirx.tensorType(mlir_ctx, sh); + } + + const frontend_attributes: mlir.Attribute = .dict(mlir_ctx, &.{}); + + const op = dialect.stablehlo.custom_call( + mlir_ctx, + custom_call_inputs[0..inputs.len], + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, + .has_side_effect = true, + .output_operand_aliases = opts.output_operand_aliases, + }, + res_types[0..res_shapes_.len], + mlir_ctx.location(@src()), + ); + + var custom_call_outputs = allocator.alloc(Tensor, 8) catch unreachable; + + for (custom_call_outputs[0..res_shapes_.len], res_shapes_, 0..) |*t, sh, i| { + t.* = Tensor._result(sh, op.result(i)); + } + + return custom_call_outputs[0..res_shapes_.len]; +} + +fn proxy(comptime custom_op: type, opts: CustomCallOptions) ffi.Handler { + return struct { + const Self = @This(); + + pub fn proxy(call_frame: *ffi.CallFrame) callconv(.C) ?*ffi.Error { + if (call_frame.registeringHook()) return null; + const execution_context = call_frame.ctx; + const user_ctx: *custom_op = ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api) catch unreachable; + const platform: Platform = user_ctx.platform; + + if (@hasField(custom_op, "stream") and platform.target != .cpu) { + const stream = call_frame.api.stream(execution_context); + user_ctx.stream = stream; + } else { + log.info("No stream field provided in container {s} for custom call", .{@typeName(custom_op)}); + } + + var callback_args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; + callback_args[0] = user_ctx; + + inline for (1..callback_args.len) |i| { + const ffi_buffer = call_frame.args.buffers()[i - 1]; + const ffi_buffer_shape = getShape(ffi_buffer); + var zml_buffer: Buffer = undefined; + + if (platform.target == .cpu) { + zml_buffer = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + zml_buffer = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch unreachable; + } + callback_args[i] = zml_buffer; + } + + for (0..call_frame.results.len) |i| { + const ffi_buffer = call_frame.results.buffers()[i]; + const ffi_buffer_shape = getShape(ffi_buffer); + + if (platform.target == .cpu) { + user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, getShape(ffi_buffer), null, ffi_buffer.data); + } + } + + @call(.auto, custom_op.call, callback_args) catch |err| { + stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(custom_op), err }); + }; + + return null; + } + }.proxy; +} + +pub fn getShape(ffi_buffer: *const ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .pred => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} + +/// Internal custom calls. +/// These are not meant to be used by users, but rather by the library itself. +pub const custom_call_internal_types = [_]type{ + Print, +}; + +pub fn registerInternalCustomCalls( + platform: Platform, +) !void { + inline for (custom_call_internal_types) |custom_call_type| { + try platform.registerFFIType(custom_call_type); + log.info("Registered internal custom call {s} with type_id {d}", .{ @typeName(custom_call_type), custom_call_type.type_id }); + } +} + +pub const Print = struct { + pub var type_id: i64 = undefined; + const Self = @This(); + + allocator: std.mem.Allocator, + platform: Platform, + + results: [1]Buffer = undefined, + + pub fn init( + allocator: std.mem.Allocator, + platform: Platform, + ) !Print { + return .{ + .allocator = allocator, + .platform = platform, + }; + } + + pub fn call(_: *Self, input: Buffer) !void { + std.log.defaultLog(.info, .zml, "Device buffer: {}: {}", .{ input.shape(), input.asHostBuffer().pretty() }); + } +}; diff --git a/zml/exe.zig b/zml/exe.zig index 0923f17c..68e631b3 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -8,6 +8,7 @@ const Bufferized = @import("tensor.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); +const custom_call = @import("custom_call.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; @@ -285,6 +286,21 @@ pub const BaseExe = struct { stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); } + pub fn bind(self: BaseExe, comptime T: type, value: *T) !void { + stdx.debug.assert(self.context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = self.platform.pjrt_api; + + if (pjrt_api.ffi()) |ffi| { + const type_id = T.type_id; + const user_data: *anyopaque = @ptrCast(@constCast(value)); + + try ffi.addUserData(pjrt_api, self.context.?, .{ .type_id = type_id, .user_data = user_data }); + log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), user_data, T.type_id, self.context.? }); + } else { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(self.platform.target)}); + } + } + pub fn serialize(self: BaseExe, writer: anytype) !void { var executable = try self.exe.getExecutable(self.platform.pjrt_api); var serialize_result = try executable.serialize(self.platform.pjrt_api); @@ -348,6 +364,28 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + pub fn withExecutionContext(self: Self) !Self { + stdx.debug.assert(self.inner.context == null, "Exe already has an execution context", .{}); + var new: Self = .{ .inner = self.inner }; + var arena = &new.inner._arena; + var allocator = arena.allocator(); + + const pjrt_execute_context = try new.inner.platform.pjrt_api.createExecuteContext(); + new.inner.context = pjrt_execute_context; + inline for (custom_call.custom_call_internal_types) |custom_call_internal_type| { + const value_ptr = try allocator.create(custom_call_internal_type); + value_ptr.* = try .init(allocator, new.platform()); + try new.inner.bind(custom_call_internal_type, value_ptr); + } + log.info("Created context execution {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); + + return new; + } + + pub fn bind(self: Self, comptime T: type, value: *T) !void { + try self.inner.bind(T, value); + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 8f6fcb1c..8a44f9af 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -176,10 +176,20 @@ pub const HostBuffer = struct { return ptr[0..self._shape.count()]; } + pub fn item(self: HostBuffer, comptime T: type, index: usize) T { + stdx.debug.assert(index < self._shape.count(), "index {} out of bounds for shape {}", .{ index, self._shape }); + return self.items(T)[index]; + } + pub fn mutItems(self: HostBuffer, comptime T: type) []T { return @constCast(self.items(T)); } + pub fn mutitem(self: HostBuffer, comptime T: type, index: usize) T { + stdx.debug.assert(index < self._shape.count(), "index {} out of bounds for shape {}", .{ index, self._shape }); + return self.mutItems(T)[index]; + } + pub fn bytes(self: HostBuffer) []const u8 { stdx.debug.assert(self.isContiguous(), "{} isn't contiguous, can't interpret as []const u8", .{self}); return self._data[0..self._shape.byteSize()]; diff --git a/zml/module.zig b/zml/module.zig index f031e815..7ad4980d 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -1157,9 +1157,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1175,7 +1175,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/ops.zig b/zml/ops.zig index e0af64e8..aaba299e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -764,33 +764,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - return customCall( - "zmlHostBufferCallback", - inputs, - output_shapes, - .{ - .callback = @intFromPtr(callback), - .user_context = @intFromPtr(blkctx), - }, - .{ - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = opts.output_operand_aliases, - }, - ); -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 1795efc3..f7783d97 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -215,6 +215,13 @@ pub const Event = opaque { return self.inner().getEventError(api); } + pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { + if (self.isReady(api)) { + return; + } + try self.inner().await_(api); + } + pub fn await_(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); diff --git a/zml/tensor.zig b/zml/tensor.zig index 74bf1779..ee0da3a4 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -13,6 +13,8 @@ const Memory = @import("buffer.zig").Buffer.Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); +const custom_call_ = @import("custom_call.zig"); +const custom_call = custom_call_.custom_call; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -3825,22 +3827,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - // TODO: find a way of doing print that doesn't involve a H2D copy. - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.log.defaultLog(.info, .zml, "Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); - // This is true because of the operand aliases. - // Since the result is already pointing to the input we don't need to modify the buffer. - std.debug.assert(host_buffer._data == outputs[0]._data); + return custom_call(custom_call_.Print, .{input}, &[_]Shape{input.shape()}, .{ .output_operand_aliases = &.{0}, .copy_inputs_to_host_pinned = true })[0]; } }; diff --git a/zml/zml.zig b/zml/zml.zig index 45abfccd..546e986c 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -34,6 +34,7 @@ pub const torch = @import("torch.zig"); pub const tokenizer = @import("zml/tokenizer"); pub const call = ops.call; +pub const custom_call = @import("custom_call.zig").custom_call; pub const compile = exe.compile; pub const compileWithPrefix = exe.compileWithPrefix; pub const compileFn = exe.compileFn; From a77bed8a060d96fd9a6d7b6e948e62bc4b3446d8 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 11 Jun 2025 15:47:50 +0000 Subject: [PATCH 12/17] async exe --- zml/context.zig | 2 -- zml/custom_call.zig | 20 ++++++++++---------- zml/exe.zig | 12 ++++++------ zml/pjrtx.zig | 12 ++++++++++-- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/zml/context.zig b/zml/context.zig index 028f88b3..772790c0 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -7,8 +7,6 @@ const runfiles = @import("runfiles"); const runtimes = @import("runtimes"); const stdx = @import("stdx"); -const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; diff --git a/zml/custom_call.zig b/zml/custom_call.zig index 1c7a2f49..4c16b54f 100644 --- a/zml/custom_call.zig +++ b/zml/custom_call.zig @@ -1,25 +1,25 @@ const std = @import("std"); const asynk = @import("async"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); const stdx = @import("stdx"); -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; -}; -const mlir = @import("mlir"); +const Buffer = @import("buffer.zig").Buffer; +const CompilationContext = @import("module.zig").CompilationContext; +const cuda = @import("context.zig").cuda; +const DataType = @import("dtype.zig").DataType; +const HostBuffer = @import("hostbuffer.zig").HostBuffer; const mlirx = @import("mlirx.zig"); -const pjrt = @import("pjrt"); const pjrtx = @import("pjrtx.zig"); const ffi = pjrtx.ffi; -const cuda = @import("context.zig").cuda; -const Buffer = @import("buffer.zig").Buffer; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; -const CompilationContext = @import("module.zig").CompilationContext; -const DataType = @import("dtype.zig").DataType; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; const log = std.log.scoped(.@"zml/custom_call"); pub const std_options: std.Options = .{ diff --git a/zml/exe.zig b/zml/exe.zig index 68e631b3..1b407c06 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -250,11 +250,11 @@ pub const BaseExe = struct { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn _unsafeAssignResults(self: BaseExe, T: type, result: *T) void { @@ -330,7 +330,7 @@ pub const BaseExe = struct { pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ - .n_in = self.input_buffer_count, + .input_shapes = self.input_shapes, .result_shapes = self.result_shapes, .n_devices = self.num_devices, }); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index f7783d97..23ed76c8 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -279,14 +279,22 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, .context = args.context, - } }); + }); + // try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + // .num_args = args.num_args, + // .arguments = @ptrCast(args.arguments), + // .results = @ptrCast(args.results), + // .events = @ptrCast(args.events), + // .non_donatable_input_indices = args.non_donatable_input_indices, + // .context = args.context, + // } }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From fc28f0dcefc9ff451e7c274e2b8e68cab764510d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 15 Jul 2025 09:58:40 +0200 Subject: [PATCH 13/17] handle memories on CPU --- zml/buffer.zig | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index b6f18fbf..7518fe09 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -89,15 +89,20 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (opts.memory) |memory| { + if (platform.target == .cpu or opts.memory == null) { + args.device = devices[i]; + } else { + const memory = opts.memory.?; const device_memories = try devices[i].addressableMemories(platform.pjrt_api); + // TODO measure the cost of this and consider caching on Zig side inside the platform. const selected_memory = for (device_memories) |m| { const kind = m.kind(platform.pjrt_api); if (kind == memory.toPjrtMemory()) break m; - } else return error.NotFound; + } else { + log.warn("Platform {s} doesn't have memory {s}", .{ @tagName(platform.target), @tagName(memory) }); + return error.NotFound; + }; args.memory = selected_memory; - } else { - args.device = devices[i]; } const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); From c7d6ffe1ca112bc9380fb22d18580b7d9e9cad57 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Jul 2025 12:20:30 +0200 Subject: [PATCH 14/17] refactor imports --- zml/context.zig | 54 ++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/zml/context.zig b/zml/context.zig index 772790c0..6a39d957 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -8,14 +8,18 @@ const runtimes = @import("runtimes"); const stdx = @import("stdx"); const pjrt = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Target = @import("platform.zig").Target; -const custom_call = @import("custom_call.zig"); -const zml_platform = @import("platform.zig"); -const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); -const PlatformsMap = std.EnumArray(Target, ?Platform); +const zml = struct { + pub const Platform = @import("platform.zig").Platform; + pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; + pub const Shape = @import("shape.zig").Shape; + pub const Target = @import("platform.zig").Target; + pub const custom_call = @import("custom_call.zig"); + pub const platform = @import("platform.zig"); +}; + +const PjrtApiMap = std.EnumArray(zml.Target, ?*const pjrt.Api); +const PlatformsMap = std.EnumArray(zml.Target, ?zml.Platform); const log = std.log.scoped(.@"zml/context"); test { @@ -93,7 +97,7 @@ pub const Context = struct { return .{ .platforms = PlatformsMap.initFill(null) }; } - fn platformToLibrary(comptime target: Target) []const u8 { + fn platformToLibrary(comptime target: zml.Target) []const u8 { const ext = switch (builtin.os.tag) { .windows => ".dll", .macos, .ios, .watchos => ".dylib", @@ -104,7 +108,7 @@ pub const Context = struct { }; } - pub fn pjrtApi(target: Target) *const pjrt.Api { + pub fn pjrtApi(target: zml.Target) *const pjrt.Api { return Context.apis.get(target).?; } @@ -118,12 +122,12 @@ pub const Context = struct { self.* = undefined; } - const prefered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; + const prefered_targets = [_]zml.Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; /// Automatically selects the best Platform loaded in the current Context. /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn autoPlatform(self: *Context, opts: Platform.CreateOptions) Platform { + pub fn autoPlatform(self: *Context, opts: zml.Platform.CreateOptions) zml.Platform { stdx.debug.assert(prefered_targets.len == apis.values.len, "New target need to be inserted inside `zml.Context.preferred_targets`", .{}); return self.platformByPreferences(opts, &prefered_targets); @@ -132,7 +136,7 @@ pub const Context = struct { /// Given a list of preferred targets to select the best Platform /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn platformByPreferences(self: *Context, opts: Platform.CreateOptions, prefered: []const Target) Platform { + pub fn platformByPreferences(self: *Context, opts: zml.Platform.CreateOptions, prefered: []const zml.Target) zml.Platform { // Try prefered targets. for (prefered) |target| { if (apis.get(target) == null) continue; @@ -149,7 +153,7 @@ pub const Context = struct { // CPU should only be use as fallback. if (target == .cpu) continue; if (entry.value.* == null) continue; - if (std.mem.indexOfScalar(Target, prefered, target) != null) continue; + if (std.mem.indexOfScalar(zml.Target, prefered, target) != null) continue; return self.platform(target, opts) catch |err| { log.err("Failed to load platform .{s}: {}", .{ @tagName(target), err }); continue; @@ -163,7 +167,7 @@ pub const Context = struct { }; } - pub fn platform(self: *Context, target: Target, opts: Platform.CreateOptions) !Platform { + pub fn platform(self: *Context, target: zml.Target, opts: zml.Platform.CreateOptions) !Platform { if (self.platforms.get(target)) |p| { return p; } @@ -181,7 +185,7 @@ pub const Context = struct { return p; } - pub fn printAvailablePlatforms(self: Context, selected: Platform) void { + pub fn printAvailablePlatforms(self: Context, selected: zml.Platform) void { // List available targets log.info("Available Platforms:", .{}); const selected_prefix = "✅"; @@ -189,7 +193,7 @@ pub const Context = struct { const selected_postfix = "(AUTO-SELECTED)"; const not_selected_postfix = ""; - for (zml_platform.available_targets) |target| { + for (zml.platform.available_targets) |target| { log.info(" {s} {s} {s}", .{ if (target == selected.target) selected_prefix else not_selected_prefix, @tagName(target), @@ -213,7 +217,7 @@ pub const Context = struct { }; const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { + pub fn registerZmlCustomCalls(platform: zml.Platform) !void { const ffi = platform.pjrt_api.ffi() orelse { log.warn("Registering custom calls failed: No FFI Extension found in {s} PJRT Plugin.", .{@tagName(platform.target)}); return; @@ -232,12 +236,12 @@ const CustomCall = struct { std.debug.assert(user_ctx_ptr.dtype == .u64); const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); + const input_buffers = stdx.stackSlice(8, zml.HostBuffer, call_frame.args.len); for (input_buffers, 0..) |*b, i| { b.* = hostBufferFromPinnedBuffer(call_frame.args.buffers()[i]); } - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); + const output_buffers = stdx.stackSlice(8, zml.HostBuffer, call_frame.results.len); for (output_buffers, 0..) |*b, i| { b.* = hostBufferFromPinnedBuffer(call_frame.results.buffers()[i]); } @@ -247,9 +251,9 @@ const CustomCall = struct { } }; -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { +fn getShape(buffer_desc: *const pjrt.ffi.Buffer) zml.Shape { // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { + const dt: zml.DataType = switch (buffer_desc.dtype) { .invalid => @panic("invalid ffi"), .pred => .bool, .i8 => .i8, @@ -257,17 +261,17 @@ fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { .i32 => .i32, .i64 => .i64, .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), + inline else => |t| @field(zml.DataType, @tagName(t)), }; - return Shape.init(buffer_desc.dims(), dt); + return zml.Shape.init(buffer_desc.dims(), dt); } /// Create a HostBuffer from a ffi description of a buffer. /// Normally the ffi describe device buffer but we assume they are located in pinned memory, /// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { +fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) zml.HostBuffer { const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( + return zml.HostBuffer.fromBytes( buffer_shape, buffer_desc.data[0..buffer_shape.byteSize()], ); From e21cd084ab47f30406c1762f25fb7941bbdef4ac Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Jul 2025 17:44:11 +0200 Subject: [PATCH 15/17] async2: handle failure on coroutine side --- async/async.zig | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/async/async.zig b/async/async.zig index 72503c22..a62aa881 100644 --- a/async/async.zig +++ b/async/async.zig @@ -73,6 +73,10 @@ fn FrameExx(comptime func: anytype, comptime argsT: type, comptime returnT: type pub fn async2(comptime func: anytype, args: anytype, result_dst: anytype) !*coro.Coro { const Sig = stdx.meta.FnSignature(func, @TypeOf(args)); var new_stack = try AsyncThread.current.stack_allocator.create(); + if (@TypeOf(result_dst) != *Sig.ReturnPayloadT) { + stdx.debug.compileError("async2 expects result_dst to be a *{s} got {s}", .{ @typeName(Sig.ReturnPayloadT), @typeName(@TypeOf(result_dst)) }); + } + // Note: I'm not a big fan of this struct. It would be simpler if res_dst was part of the Coro itself. const storage = try new_stack.push(struct { res_dst: *anyopaque, args: Sig.ArgsT }); storage.args = args; @@ -94,11 +98,12 @@ fn Helpers_async2(comptime Sig: stdx.meta.Signature) type { const storage: *struct { res_dst: *anyopaque, args: Sig.ArgsT } = @alignCast(@ptrCast(storage_raw)); const res_dst: *Sig.ReturnT = @alignCast(@ptrCast(storage.res_dst)); - res_dst.* = @call( - .auto, - Sig.Func.Value, - storage.args, - ); + const res = @call(.auto, Sig.Func.Value, storage.args); + if (Sig.ReturnErrorSet) |_| { + res_dst.* = res catch |err| std.debug.panic("coroutine failed: {} with {}", .{ Sig.Func, err }); + } else { + res_dst.* = res; + } } }; } From 2f071498859b378eeb5da3011c667d19ba206bbc Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Jul 2025 17:44:37 +0200 Subject: [PATCH 16/17] ExecutionContext.Context(T).get() -> ExecutionContext.get(T) --- pjrt/ffi.zig | 37 +++++++++++++++++-------------------- zml/custom_call.zig | 41 +++++++++++++++++++++-------------------- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 3d6376ad..24a2cae7 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -163,29 +163,26 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - pub fn Context(comptime T: type) type { - return struct { - pub fn get(self: *const ExecutionContext, api: *const Api) ApiError!*T { - const type_id: TypeId = .{ .type_id = T.type_id }; - var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = @constCast(self.inner()), - .type_id = @constCast(&type_id), - }); - const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + pub fn getUserDataRaw(self: *const ExecutionContext, api: *const Api, type_id: i64) ?*anyopaque { + var type_id_struct: TypeId = .{ .type_id = type_id }; + var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ + .ctx = @constCast(self.inner()), + .type_id = &type_id_struct, + }); + const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); - if (result) |ffi_error| { - const err = Error.fromInner(ffi_error); - defer err.destroy(api); - log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); - // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - return error.Unknown; - } + return null; + } + return ret.data; + } - if (ret.data == null) return error.NotFound; - return @ptrCast(@alignCast(ret.data.?)); - } - }; + pub fn getUserData(self: *const ExecutionContext, T: type, api: *const Api) ?*T { + return @alignCast(@ptrCast(self.getUserDataRaw(api, T.type_id))); } pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { diff --git a/zml/custom_call.zig b/zml/custom_call.zig index 4c16b54f..a0da18a4 100644 --- a/zml/custom_call.zig +++ b/zml/custom_call.zig @@ -27,8 +27,8 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; -pub fn CustomCallInputsType(custom_op: type) type { - const ArgsT = stdx.meta.FnArgs(custom_op.call); +pub fn CustomCallInputsType(CustomOp: type) type { + const ArgsT = stdx.meta.FnArgs(CustomOp.call); if (@typeInfo(ArgsT) != .@"struct") { @compileError("Expected struct type"); } @@ -53,15 +53,16 @@ pub const CustomCallOptions = struct { }; pub fn custom_call( - comptime custom_op: type, - inputs: CustomCallInputsType(custom_op), + comptime CustomOp: type, + inputs: CustomCallInputsType(CustomOp), res_shapes_: []const Shape, comptime opts: CustomCallOptions, ) []Tensor { - stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); - const op_name = @typeName(custom_op); + stdx.debug.assert(@hasDecl(CustomOp, "call"), "CustomOp must have a call method", .{}); + const op_name = @typeName(CustomOp); const ctx = CompilationContext.current(); + ctx._num_custom_calls += 1; const allocator = ctx.allocator(); const mlir_ctx = ctx.mlirCtx(); const platform = ctx._platform; @@ -72,7 +73,7 @@ pub fn custom_call( stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); } - const ffi_func = proxy(custom_op, opts); + const ffi_func = proxy(CustomOp, opts); const target_name = "callback_" ++ op_name; ffi_.?.register(pjrt_api, target_name, @tagName(platform.target), &ffi_func, opts.register_ffi_options) catch unreachable; @@ -114,24 +115,28 @@ pub fn custom_call( return custom_call_outputs[0..res_shapes_.len]; } -fn proxy(comptime custom_op: type, opts: CustomCallOptions) ffi.Handler { +fn proxy(comptime CustomOp: type, opts: CustomCallOptions) ffi.Handler { + stdx.debug.assertComptime(@hasField(CustomOp, "platform"), "CustomCall context struct needs to contain a 'platform: zml.Platform' field in struct {s}", .{@typeName(CustomOp)}); + return struct { const Self = @This(); pub fn proxy(call_frame: *ffi.CallFrame) callconv(.C) ?*ffi.Error { if (call_frame.registeringHook()) return null; const execution_context = call_frame.ctx; - const user_ctx: *custom_op = ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api) catch unreachable; + const user_ctx: *CustomOp = execution_context.getUserData(CustomOp, call_frame.api) orelse + stdx.debug.panic("An exe was called but its execution context {s} was never bound", .{@typeName(CustomOp)}); const platform: Platform = user_ctx.platform; + log.debug("Callback called with ffi api: {*} but pjrt_api: {*}", .{ call_frame.api, platform.pjrt_api }); - if (@hasField(custom_op, "stream") and platform.target != .cpu) { + if (@hasField(CustomOp, "stream") and platform.target != .cpu) { const stream = call_frame.api.stream(execution_context); user_ctx.stream = stream; } else { - log.info("No stream field provided in container {s} for custom call", .{@typeName(custom_op)}); + log.info("No stream field provided in container {s} for custom call", .{@typeName(CustomOp)}); } - var callback_args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; + var callback_args: std.meta.ArgsTuple(@TypeOf(CustomOp.call)) = undefined; callback_args[0] = user_ctx; inline for (1..callback_args.len) |i| { @@ -161,8 +166,8 @@ fn proxy(comptime custom_op: type, opts: CustomCallOptions) ffi.Handler { } } - @call(.auto, custom_op.call, callback_args) catch |err| { - stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(custom_op), err }); + @call(.auto, CustomOp.call, callback_args) catch |err| { + stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(CustomOp), err }); }; return null; @@ -203,17 +208,13 @@ pub const Print = struct { pub var type_id: i64 = undefined; const Self = @This(); - allocator: std.mem.Allocator, + // TODO: is this a good idea to use the platform to platform: Platform, results: [1]Buffer = undefined, - pub fn init( - allocator: std.mem.Allocator, - platform: Platform, - ) !Print { + pub fn init(platform: Platform) Print { return .{ - .allocator = allocator, .platform = platform, }; } From b06f9fb99469769614a9c65a4cb2ac97768d6593 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 22 Jul 2025 18:27:55 +0200 Subject: [PATCH 17/17] automatically create ExecuteContext when needed by executable. --- pjrt/pjrt.zig | 7 +++--- zml/exe.zig | 58 +++++++++++++++++++++++------------------------- zml/module.zig | 2 ++ zml/platform.zig | 1 + 4 files changed, 34 insertions(+), 34 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 29a7271d..fe37ab81 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1298,15 +1298,14 @@ pub const NamedValue = extern struct { pub const FFI = extern struct { inner: *const c.PJRT_FFI, + // we clone the struct to force fields to be initialized by the caller. pub const UserData = extern struct { type_id: i64, user_data: *anyopaque, + deleter: ?*fn (*anyopaque) void = null, fn toCStruct(self: UserData) c.PJRT_FFI_UserData { - return .{ - .type_id = self.type_id, - .data = self.user_data, - }; + return @bitCast(self); } }; diff --git a/zml/exe.zig b/zml/exe.zig index 492eeb02..7761c1c2 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -155,7 +155,7 @@ pub const BaseExe = struct { exe: *pjrt.LoadedExecutable, /// The execution context for this executable. - context: ?*pjrt.ExecuteContext = null, + exe_context: ?*pjrt.ExecuteContext = null, /// Pre-allocated slice of buffers to use as inputs when the module is called. input_per_device: []const [*]*pjrt.Buffer, @@ -182,7 +182,7 @@ pub const BaseExe = struct { parent_allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, - args: struct { input_shapes: []const Shape, result_shapes: []const Shape, n_devices: u8 }, + args: struct { input_shapes: []const Shape, result_shapes: []const Shape, n_devices: u8, need_execution_context: bool }, ) !BaseExe { var arena = std.heap.ArenaAllocator.init(parent_allocator); errdefer arena.deinit(); @@ -206,7 +206,8 @@ pub const BaseExe = struct { const all_shapes = try allocator.alloc(Shape, n_in + n_out); @memcpy(all_shapes[0..n_in], args.input_shapes); @memcpy(all_shapes[n_in..], args.result_shapes); - return .{ + + var base_exe: BaseExe = .{ .platform = platform, .exe = exe, .ready_buffer_count = 0, @@ -216,12 +217,20 @@ pub const BaseExe = struct { .output_per_device = output_per_device, .input_shapes = all_shapes[0..n_in], .result_shapes = all_shapes[n_in..], + .exe_context = if (args.need_execution_context) try platform.pjrt_api.createExecuteContext() else null, ._arena = arena, }; + // Those may not needed, but also it's annoying if they don't get registered automatically. + if (args.need_execution_context) { + inline for (custom_call.custom_call_internal_types) |T| { + try base_exe.bind(T, .init(platform)); + } + } + return base_exe; } pub fn deinit(self: BaseExe) void { - if (self.context) |ctx| { + if (self.exe_context) |ctx| { ctx.deinit(self.platform.pjrt_api); } self._arena.deinit(); @@ -245,7 +254,7 @@ pub const BaseExe = struct { // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, - .context = self.context, + .context = self.exe_context, }) catch |err| { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; @@ -286,16 +295,19 @@ pub const BaseExe = struct { stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); } - pub fn bind(self: BaseExe, comptime T: type, value: *T) !void { - stdx.debug.assert(self.context != null, "Exe doesn't have an execution context", .{}); + pub fn bind(self: *BaseExe, comptime T: type, value: T) !void { + const storage = try self._arena.allocator().create(T); + storage.* = value; + try self.bindPtr(T, storage); + } + + pub fn bindPtr(self: BaseExe, comptime T: type, user_data: *T) !void { + stdx.debug.assert(self.exe_context != null, "Exe doesn't have an execution exe_context", .{}); const pjrt_api = self.platform.pjrt_api; if (pjrt_api.ffi()) |ffi| { - const type_id = T.type_id; - const user_data: *anyopaque = @ptrCast(@constCast(value)); - - try ffi.addUserData(pjrt_api, self.context.?, .{ .type_id = type_id, .user_data = user_data }); - log.info("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), user_data, T.type_id, self.context.? }); + try ffi.addUserData(pjrt_api, self.exe_context.?, .{ .type_id = T.type_id, .user_data = user_data }); + log.debug("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(T), @intFromPtr(user_data), T.type_id, self.exe_context.? }); } else { stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(self.platform.target)}); } @@ -364,26 +376,12 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } - pub fn withExecutionContext(self: Self) !Self { - stdx.debug.assert(self.inner.context == null, "Exe already has an execution context", .{}); - var new: Self = .{ .inner = self.inner }; - var arena = &new.inner._arena; - var allocator = arena.allocator(); - - const pjrt_execute_context = try new.inner.platform.pjrt_api.createExecuteContext(); - new.inner.context = pjrt_execute_context; - inline for (custom_call.custom_call_internal_types) |custom_call_internal_type| { - const value_ptr = try allocator.create(custom_call_internal_type); - value_ptr.* = try .init(allocator, new.platform()); - try new.inner.bind(custom_call_internal_type, value_ptr); - } - log.info("Created context execution {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); - - return new; + pub fn bind(self: *Self, comptime T: type, value: T) !void { + try self.inner.bind(T, value); } - pub fn bind(self: Self, comptime T: type, value: *T) !void { - try self.inner.bind(T, value); + pub fn bindPtr(self: Self, comptime T: type, value: *T) !void { + try self.inner.bindPtr(T, value); } pub fn serialize(self: Self, writer: anytype) !void { diff --git a/zml/module.zig b/zml/module.zig index b5bead0c..8eb64f77 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -49,6 +49,7 @@ pub const CompilationContext = struct { _mlir_canonicalizer: mlir.PassManager, _module: mlir.Module, + _num_custom_calls: u32 = 0, _blocks: std.BoundedArray(TaggedBlock, 64) = .{}, _fn_cache: FnCache = .{}, @@ -233,6 +234,7 @@ pub const CompilationContext = struct { .input_shapes = f.args_shapes, .result_shapes = f.res_shapes, .n_devices = sharding.num_replicas * sharding.num_partitions, + .need_execution_context = self._num_custom_calls > 0, }, ); } diff --git a/zml/platform.zig b/zml/platform.zig index a7bc87c9..ea197b4f 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -53,6 +53,7 @@ pub const Platform = struct { pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; pub fn sharding(self: Platform) Sharding { + // TODO: store me inside the platform itself to avoid expensive ffi calls // replicas run the same function but with different inputs, // while partitions contribute to one evaluation over a shared input. // Inside an inference process, we generally don't want replicas,