From 862b654668d82e397530f52b1fe4eaca1f8fe0fd Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 3 Feb 2026 17:09:21 +0100 Subject: [PATCH 1/6] platforms: add metal --- examples/mnist/mnist.zig | 2 ++ platforms/BUILD.bazel | 1 + platforms/metal/BUILD.bazel | 47 ++++++++++++++++++++++++++++++ platforms/metal/metal.zig | 58 +++++++++++++++++++++++++++++++++++++ platforms/platforms.zig | 4 +++ zml/io.zig | 2 +- zml/mem.zig | 2 +- zml/platform.zig | 27 +++++++++++++---- 8 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 platforms/metal/BUILD.bazel create mode 100644 platforms/metal/metal.zig diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index fdaa6900..989b405f 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -94,6 +94,8 @@ pub fn main() !void { // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + log.info("\n{f}", .{platform.fmtVerbose()}); // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index a2a023b1..c965282f 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -7,6 +7,7 @@ PLATFORMS = { "rocm": False, "tpu": False, "neuron": False, + "metal": False, } [ diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel new file mode 100644 index 00000000..aa2c0c6a --- /dev/null +++ b/platforms/metal/BUILD.bazel @@ -0,0 +1,47 @@ +load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "empty", + defines = ["ZML_RUNTIME_METAL_DISABLED"], +) + +copy_to_directory( + name = "sandbox", + srcs = [ + "libIREECompiler.so", + "pjrt_plugin_iree_metal.dylib", + "@toolchains_llvm_bootstrapped//tools:lld", + ], + add_directory_to_runfiles = False, + include_external_repositories = ["**"], + replace_prefixes = { + "pjrt_plugin_iree_metal.dylib": "lib/pjrt_plugin_iree_metal.dylib", + "libIREECompiler.so": "lib/libIREECompiler.so", + }, +) + +cc_library( + name = "libpjrt_metal", + data = [":sandbox"], + defines = ["ZML_RUNTIME_METAL"], +) + +zig_library( + name = "metal", + import_name = "platforms/metal", + main = "metal.zig", + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + ] + select({ + "//platforms:metal.enabled": [ + ":libpjrt_metal", + "//bazel", + "//stdx", + "@rules_zig//zig/runfiles", + ], + "//conditions:default": [":empty"], + }), +) diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig new file mode 100644 index 00000000..67732b63 --- /dev/null +++ b/platforms/metal/metal.zig @@ -0,0 +1,58 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const bazel_runfiles = @import("runfiles"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/metal"); + +pub fn isEnabled() bool { + return @hasDecl(c, "ZML_RUNTIME_METAL"); +} + +fn setupEnv(sandbox_path: []const u8) !void { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + _ = c.setenv( + "IREE_PJRT_COMPILER_LIB_PATH", + @ptrCast(try stdx.Io.Dir.path.bufJoinZ(&path_buf, &.{ sandbox_path, "lib/libIREECompiler.so" })), + 1, + ); // must be zero terminated + _ = c.setenv( + "IREE_LLVM_EMBEDDED_LINKER_PATH", + @ptrCast(try stdx.Io.Dir.path.bufJoinZ(&path_buf, &.{ sandbox_path, "bin/lld" })), + 1, + ); +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = allocator; // autofix + if (comptime !isEnabled()) { + return error.Unavailable; + } + if (comptime builtin.os.tag != .macos) { + return error.Unavailable; + } + + const r = try bazel.runfiles(io, bazel_builtin.current_repository); + + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = (r.rlocation("zml/platforms/metal/sandbox", &sandbox_path_buf) catch |err| { + log.err("unable to resolve sandbox path: {any}", .{err}); + return err; + }) orelse { + log.err("unable to resolve sandbox path", .{}); + return error.Unavailable; + }; + + try setupEnv(sandbox_path); + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "pjrt_plugin_iree_metal.dylib" }); + break :blk pjrt.Api.loadFrom(path); + }; +} diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 4033415c..67d78a82 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -2,6 +2,7 @@ const std = @import("std"); const cpu = @import("platforms/cpu"); const cuda = @import("platforms/cuda"); +const metal = @import("platforms/metal"); const neuron = @import("platforms/neuron"); const pjrt = @import("pjrt"); const rocm = @import("platforms/rocm"); @@ -13,6 +14,7 @@ pub const Platform = enum { rocm, tpu, neuron, + metal, }; pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjrt.Api { @@ -22,6 +24,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjr .rocm => try rocm.load(allocator, io), .tpu => try tpu.load(allocator, io), .neuron => try neuron.load(allocator, io), + .metal => try metal.load(allocator, io), }; } @@ -32,5 +35,6 @@ pub fn isEnabled(tag: Platform) bool { .rocm => rocm.isEnabled(), .tpu => tpu.isEnabled(), .neuron => neuron.isEnabled(), + .metal => metal.isEnabled(), }; } diff --git a/zml/io.zig b/zml/io.zig index 814b4e6a..e199b9d9 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -274,7 +274,7 @@ pub const MemoryWriter = union(enum) { pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !MemoryWriter { return switch (memory.platform.target) { .cuda, .rocm => .{ .direct = try .init(allocator, io, memory, pool, shape, buffer) }, - .tpu, .neuron, .cpu => .{ .buffered = try .init(allocator, io, memory, shape, buffer) }, + .tpu, .neuron, .cpu, .metal => .{ .buffered = try .init(allocator, io, memory, shape, buffer) }, }; } diff --git a/zml/mem.zig b/zml/mem.zig index 68c7bc74..1f7be3de 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -23,7 +23,7 @@ pub const DmaAllocator = union(enum) { return switch (device.platform.target) { .cuda, .rocm => .{ .dmam = .init(parent, device.platform) }, .tpu, .neuron => .{ .uib = .init(device.memory(.host_pinned)) }, - .cpu => .{ .passthrough = parent }, + .cpu, .metal => .{ .passthrough = parent }, }; } diff --git a/zml/platform.zig b/zml/platform.zig index 346c95e1..af081729 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -76,7 +76,11 @@ pub const Memory = struct { const pjrt_addressable_by_devices = pjrt_memory.addressableByDevices(platform.pjrt_api); const addressable_by_devices = try allocator.alloc(*const Device, pjrt_addressable_by_devices.len); for (pjrt_addressable_by_devices, addressable_by_devices) |pjrt_device, *addressable_by_device| { - addressable_by_device.* = &all_devices[pjrt_device.localHardwareId(platform.pjrt_api)]; + const pjrt_device_ordinal = switch (platform.target) { + .cuda, .rocm, .tpu, .neuron, .cpu => pjrt_device.localHardwareId(platform.pjrt_api), + .metal => 0, + }; + addressable_by_device.* = &all_devices[pjrt_device_ordinal]; } return .{ @@ -101,7 +105,7 @@ pub const Memory = struct { }; return zml_kind == kind_; }, - .cpu, .tpu, .neuron => return true, + .cpu, .tpu, .neuron, .metal => return true, } } @@ -120,7 +124,11 @@ pub const Device = struct { const pjrt_addressable_memories = pjrt_device_.addressableMemories(platform.pjrt_api); const addressable_memories = try allocator.alloc(*const Memory, pjrt_addressable_memories.len); for (pjrt_addressable_memories, addressable_memories) |pjrt_memory, *addressable_memory| { - addressable_memory.* = &all_addressable_memories[pjrt_memory.id(platform.pjrt_api)]; + const pjrt_memory_ordinal = switch (platform.target) { + .cuda, .rocm, .tpu, .neuron, .cpu => pjrt_memory.id(platform.pjrt_api), + .metal => 0, + }; + addressable_memory.* = &all_addressable_memories[pjrt_memory_ordinal]; } return .{ @@ -144,7 +152,10 @@ pub const Device = struct { } pub fn localHardwareId(self: Device) i32 { - return self.device.localHardwareId(self.platform.pjrt_api); + return switch (self.platform.target) { + .cuda, .rocm, .tpu, .neuron, .cpu => self.device.localHardwareId(self.platform.pjrt_api), + .metal => 0, + }; } pub fn kind(self: Device) []const u8 { @@ -173,7 +184,11 @@ pub const Device = struct { pub fn memory(self: Device, memory_kind: Memory.Kind) *const Memory { if (memory_kind == .default) { const mem = self.pjrt_device.defaultMemory(self.platform.pjrt_api); - return &self.platform.memories[mem.id(self.platform.pjrt_api)]; + const pjrt_memory_ordinal = switch (self.platform.target) { + .cuda, .rocm, .tpu, .neuron, .cpu => mem.id(self.platform.pjrt_api), + .metal => 0, + }; + return &self.platform.memories[pjrt_memory_ordinal]; } for (self.addressable_memories) |mem| { @@ -261,6 +276,7 @@ pub const Platform = struct { .neuron, .rocm, .cuda, + .metal, .cpu, }; return for (ordered_targets) |target| { @@ -375,6 +391,7 @@ pub const CreateOptions = struct { rocm: struct {} = .{}, tpu: struct {} = .{}, neuron: struct {} = .{}, + metal: struct {} = .{}, pub const Cpu = struct { device_count: u32, From fb3778e619016562a2641eff12ccc0eb581519a2 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 3 Feb 2026 20:05:27 +0100 Subject: [PATCH 2/6] wip --- examples/llama/llama.zig | 11 ++++++----- examples/llama/main.zig | 8 +++++++- pjrt/pjrt.zig | 38 +++++++++++++++++++++----------------- zml/buffer.zig | 22 ++++++++++++++++------ 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index cdc0f1be..049dd856 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -119,8 +119,9 @@ pub const LlamaLM = struct { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache, attention_metadata, attention_parameters); - const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + return .{ out, updated_kv_cache, rng }; + // const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + // return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } pub fn sampleTokens( @@ -409,8 +410,8 @@ pub const SelfAttn = struct { x: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + _: zml.attention.Metadata, + _: zml.attention.Parameters, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); @@ -436,7 +437,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = q; //zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 09d1f861..0134e91a 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -460,6 +460,8 @@ pub fn generateText( var generated_token_slice: zml.Slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)); defer generated_token_slice.free(allocator); +std.log.info("prompt tok len: {}", .{prompt_tok.len}); +std.log.info("prompt len: {}", .{prompt.len}); { var prefill_args = try prefill_exe.args(allocator); defer prefill_args.deinit(allocator); @@ -479,13 +481,16 @@ pub fn generateText( prefill_args.set(.{ llama_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); +std.log.info("Before", .{}); prefill_exe.call(prefill_args, &prefill_results); +std.log.info("After", .{}); prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); +std.log.info("After event", .{}); generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; } - + std.log.info("Prefill generated: {d}", .{generated_token_slice.items(u32)[0]}); // Prepare for token-by-token generation, var decode_args = try decode_exe.args(allocator); @@ -508,6 +513,7 @@ pub fn generateText( // collect and print generated sequence num_tokens_generated += 1; const generated_token = generated_token_slice.items(u32)[0]; + std.log.info("Decode generated: {d}", .{generated_token_slice.items(u32)[0]}); if (try tokenizer_decoder.next(generated_token)) |chunk| { try writer.writeAll(chunk); try writer.flush(); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index d68cff1b..a081b465 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -234,23 +234,24 @@ pub const Api = struct { }; } - pub fn stablehloCurrentVersion(self: *const Api) ?[]const u8 { - const state = struct { - var buf: [32]u8 = undefined; - var str: ?[:0]const u8 = null; - }; - if (state.str) |str| { - return str; - } - if (self.getPluginAttribute("stablehlo_current_version")) |nv| { - switch (nv.value()) { - .int64list => |v| { - state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; - }, - else => unreachable, - } - } - return state.str; + pub fn stablehloCurrentVersion(_: *const Api) ?[]const u8 { + return "1.13.0"; + // const state = struct { + // var buf: [32]u8 = undefined; + // var str: ?[:0]const u8 = null; + // }; + // if (state.str) |str| { + // return str; + // } + // if (self.getPluginAttribute("stablehlo_current_version")) |nv| { + // switch (nv.value()) { + // .int64list => |v| { + // state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; + // }, + // else => unreachable, + // } + // } + // return state.str; } pub fn createExecuteContext(api: *const Api) ApiError!*ExecuteContext { @@ -1149,6 +1150,9 @@ pub const Event = opaque { } pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { + if (true) { + return; + } if (self.isReady(api)) { return; } diff --git a/zml/buffer.zig b/zml/buffer.zig index 7b0a88ac..0d7df801 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -86,7 +86,7 @@ pub const Buffer = struct { .buffer_type = buffer_type, .dims = shape_.dims(), .byte_strides = byte_strides.slice(), - .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .host_buffer_semantics = .ImmutableOnlyDuringCall, .dst = .{ .memory = devices[i].memory(opts.memory).pjrt_memory }, }; @@ -96,9 +96,10 @@ pub const Buffer = struct { res._shards.appendAssumeCapacity(pjrt_buffer); } - if (opts.wait) { - try res.await(io); - } + _ = io; + // if (opts.wait) { + // try res.await(io); + // } return res; } @@ -136,7 +137,12 @@ pub const Buffer = struct { pub const UnitializedOptions = struct { memory: Memory.Kind = .default }; pub fn uninitialized(io: std.Io, platform: *const Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { - _ = io; // autofix + if (platform.target == .metal) { + const slice: Slice = try .alloc(std.heap.c_allocator, shape_); + defer slice.free(std.heap.c_allocator); + + return .fromSliceOpts(io, platform, slice, .{}); + } var res: Buffer = .{ .platform = platform, ._shape = shape_, @@ -212,8 +218,12 @@ pub const Buffer = struct { pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self.platform.pjrt_api, slice.data()); + _ = io; if (maybe_event) |event| { - try event.await(self.platform.pjrt_api, io); + event.deinit(self.platform.pjrt_api); + // std.log.info("Waiting for toSlice event", .{}); + // try event.await(self.platform.pjrt_api, io); + // std.log.info("Waited for toSlice event", .{}); } } From 30c25d92941ccef95ccf51c378ea8d4033e4df8b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 4 Feb 2026 09:47:41 +0100 Subject: [PATCH 3/6] wip --- examples/llama/llama.zig | 11 +++++------ pjrt/pjrt.zig | 3 --- zml/attention/attention.zig | 2 +- zml/buffer.zig | 5 +---- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 049dd856..2d9268eb 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -119,9 +119,8 @@ pub const LlamaLM = struct { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache, attention_metadata, attention_parameters); - return .{ out, updated_kv_cache, rng }; - // const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); - // return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } pub fn sampleTokens( @@ -410,8 +409,8 @@ pub const SelfAttn = struct { x: Tensor, token_index: Tensor, kv_cache: KvCache, - _: zml.attention.Metadata, - _: zml.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); @@ -437,7 +436,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = q; //zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index a081b465..8a213ffa 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1150,9 +1150,6 @@ pub const Event = opaque { } pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { - if (true) { - return; - } if (self.isReady(api)) { return; } diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 34bf52e0..bf6e3c79 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -111,7 +111,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T // Note: in Pytorch it would be very inefficient to generate the full attn_mask, // then slice into it, but XLA is able to optimize this correctly. attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = null, .allow_cudnn = true }); break :b attn_output; }, .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), diff --git a/zml/buffer.zig b/zml/buffer.zig index 0d7df801..0a83bcb6 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -218,12 +218,9 @@ pub const Buffer = struct { pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self.platform.pjrt_api, slice.data()); - _ = io; if (maybe_event) |event| { + try event.await(self.platform.pjrt_api, io); event.deinit(self.platform.pjrt_api); - // std.log.info("Waiting for toSlice event", .{}); - // try event.await(self.platform.pjrt_api, io); - // std.log.info("Waited for toSlice event", .{}); } } From 161ff5defc2d30e7f88969ed735c29e107447132 Mon Sep 17 00:00:00 2001 From: louis Date: Tue, 3 Feb 2026 19:03:51 +0000 Subject: [PATCH 4/6] llama per layers --- examples/llama/BUILD.bazel | 12 + examples/llama/main_layers.zig | 717 +++++++++++++++++++++++++++++++++ 2 files changed, 729 insertions(+) create mode 100644 examples/llama/main_layers.zig diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index c8fce3bd..54eb7235 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -20,6 +20,18 @@ zig_binary( deps = ["//zml"], ) +zig_binary( + name = "llama_layers", + main = "main_layers.zig", + srcs = [ + "llama.zig" + ], + deps = [ + "@zml//zml", + "@zml//stdx", + ], +) + mtree_spec( name = "mtree", diff --git a/examples/llama/main_layers.zig b/examples/llama/main_layers.zig new file mode 100644 index 00000000..38f704f7 --- /dev/null +++ b/examples/llama/main_layers.zig @@ -0,0 +1,717 @@ +const std = @import("std"); +const log = std.log; + +const zml = @import("zml"); +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; +const stdx = zml.stdx; + +const llama = @import("llama.zig"); +const LlamaLM = llama.LlamaLM; +const Llama = llama.Llama; +const KvCache = llama.KvCache; +const TransformerLayer = llama.TransformerLayer; +const SelfAttn = llama.SelfAttn; + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const CliArgs = struct { + model: []const u8, + prompt: ?[]const u8 = null, + seqlen: u32 = 512, + reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, + writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, + async_limit: ?u32 = null, + backend: ?zml.attention.Backend = null, + + pub const help = + \\Usage: llama --model= [options] + \\ + \\Run LLaMA inference on a model. + \\ + \\Options: + \\ --model= Path to model (local path or HuggingFace repo) + \\ --prompt= Input prompt (reads from stdin if not provided) + \\ --seqlen= Maximum sequence length (default: 512) + \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) + \\ --reader-buffer-size= Reader buffer size (default: 32MiB) + \\ --writer-buffer-size= Writer buffer size (default: 32MiB) + \\ --async-limit= Async I/O concurrency limit + \\ -h, --help Show this help message + \\ + \\Examples: + \\ llama --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="Hello, world!" + \\ llama --model=$(realpath ../Llama-3.1-8B-Instruct) --seqlen=1024 + \\ echo "What is 2+2?" | llama --model=meta-llama/Llama-3.1-8B-Instruct + \\ + ; +}; + +pub fn main() !void { + log.info("LLama was compiled with {}", .{@import("builtin").mode}); + + // var debug_allocator: ?std.heap.DebugAllocator(.{}) = null; + // const allocator = if (@import("builtin").mode == .Debug) blk: { + // debug_allocator = .init; + // break :blk debug_allocator.?.allocator(); + // } else std.heap.c_allocator; + // defer if (debug_allocator) |*da| std.debug.assert(da.deinit() == .ok); + const allocator = std.heap.c_allocator; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const args = stdx.flags.parseProcessArgs(CliArgs); + + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + defer vfs.deinit(); + + var http_client: std.http.Client = .{ + .allocator = allocator, + .io = threaded.io(), + }; + + try http_client.initDefaultProxies(allocator); + defer http_client.deinit(); + + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); + defer vfs_file.deinit(); + try vfs.register("file", vfs_file.io()); + + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .https); + defer vfs_https.deinit(); + try vfs.register("https", vfs_https.io()); + + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); + defer hf_vfs.deinit(); + try vfs.register("hf", hf_vfs.io()); + + const io = vfs.io(); + + log.info("Resolving model repo", .{}); + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + + const parsed_config = try parseConfig(allocator, io, repo); + defer parsed_config.deinit(); + const config = parsed_config.value; + + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + // Write metadata from the config file into the LlamaLm struct. + const llama_options: llama.LlamaLM.Options = .{ + .max_seq_len = args.seqlen, + .sampling_strategy = .{ + .topk = 2, + .temperature = 1.0, + }, + }; + + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + log.info("\n{f}", .{platform.fmtVerbose()}); + // log.info("Devices:", .{}); + // for (platform.devices) |device| { + // log.info("\t- {f}", .{device}); + // } + + // Initialize the Llama struct and map the content of the .safetensors to the model tensors. + var llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); + defer llama_model.deinit(allocator); + + const backend = args.backend orelse b: { + const selected = zml.attention.Backend.auto(platform); + log.info("Selected backend: {}", .{selected}); + break :b selected; + }; + + // Specify shapes of input arguments + const dtype = llama_model.model.embed_tokens.weight.dtype(); + const llama_parameters: LlamaParameters = .{ + .prefill_tokens = .init(.{ .s = llama_options.max_seq_len }, .i32), + .decode_tokens = .init(.{ .s = 1 }, .i32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(.init(.{ + .layer = llama_model.model.layers.len, + .k = args.seqlen, + .h = config.num_key_value_heads, + .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), + }, dtype)), + .rng = .init(), + .attention_metadata = .init(.fromBackend(backend, @intCast(args.seqlen))), + .attention_parameters = .init(.fromBackend(backend)), + }; + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + + var tokenizer_future = try io.concurrent(loadTokenizer, .{ allocator, io, repo, &progress }); + errdefer blk: { + var v = tokenizer_future.cancel(io) catch break :blk; + v.deinit(); + } + + var compiled_model_result_future = try io.concurrent(compile, .{ allocator, io, platform, llama_model, llama_parameters, &progress }); + errdefer if (compiled_model_result_future.cancel(io)) |v| { + defer v.embedding_prefill_exe.deinit(); + defer v.embedding_decode_exe.deinit(); + defer v.layer_prefill_exe.deinit(); + defer v.layer_decode_exe.deinit(); + defer v.sampling_prefill_exe.deinit(); + defer v.sampling_decode_exe.deinit(); + } else |_| {}; + + var load_group: stdx.Io.LimitedGroup = .init(16); + defer load_group.cancel(io); + + var llama_buffers_future = try io.concurrent(LlamaLM.load, .{ + &llama_model, + allocator, + io, + platform, + &store, + &progress, + }); + errdefer b: { + var v = llama_buffers_future.cancel(io) catch break :b; + v = v; // autofix + // LlamaLM.unloadBuffers(&v, allocator); + } + + const compiled_model_result = try compiled_model_result_future.await(io); + defer compiled_model_result.embedding_prefill_exe.deinit(); + defer compiled_model_result.embedding_decode_exe.deinit(); + defer compiled_model_result.layer_prefill_exe.deinit(); + defer compiled_model_result.layer_decode_exe.deinit(); + defer compiled_model_result.sampling_prefill_exe.deinit(); + defer compiled_model_result.sampling_decode_exe.deinit(); + + var llama_buffers = try llama_buffers_future.await(io); + llama_buffers = llama_buffers; // autofix + // defer LlamaLM.unloadBuffers(&llama_buffers, allocator); + progress.end(); + + log.info("Creating KvCache", .{}); + var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); + defer llama.KvCache.deinitBuffer(&kv_cache_buffers); + + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + var tokenizer = try tokenizer_future.await(io); + defer tokenizer.deinit(); + + const prompt = if (args.prompt) |p| p else blk: { + var reader = std.Io.File.stdin().reader(io, &.{}); + break :blk try reader.interface.allocRemaining(allocator, .unlimited); + }; + + log.info("✅ Prompt: {s}", .{prompt}); + + // Unbuffered writing of the tokens to stdout. + var stdout = std.Io.File.stdout().writer(io, &.{}); + + try generateText( + allocator, + io, + &llama_buffers, + compiled_model_result.embedding_prefill_exe, + compiled_model_result.embedding_decode_exe, + compiled_model_result.layer_prefill_exe, + compiled_model_result.layer_decode_exe, + compiled_model_result.norm_prefill_exe, + compiled_model_result.norm_decode_exe, + compiled_model_result.sampling_prefill_exe, + compiled_model_result.sampling_decode_exe, + &kv_cache_buffers, + &attention_metadata_buffers, + tokenizer, + config, + llama_options, + @intCast((try std.Io.Clock.now(.real, io)).toNanoseconds()), + prompt[0..], + false, + &stdout.interface, + platform, + ); +} + +fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(LlamaLM.Config) { + var timer = try std.time.Timer.start(); + log.info("Loading model config", .{}); + defer log.info("Loaded model config [{D}]", .{timer.read()}); + + const parsed_config = blk: { + const config_json_file = try dir.openFile(io, "config.json", .{}); + defer config_json_file.close(io); + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(io, &config_json_buffer); + var reader: std.json.Reader = .init(allocator, &config_reader.interface); + defer reader.deinit(); + break :blk try std.json.parseFromTokenSource(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + }; + errdefer parsed_config.deinit(); + + return parsed_config; +} + +fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { + // log.info("Loading tokenizer", .{}); + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Loading tokenizer...", 1); + defer node.end(); + var timer = try std.time.Timer.start(); + defer log.info("Loaded tokenizer [{D}]", .{timer.read()}); + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + + return try .fromBytes(allocator, io, bytes); +} + +const LlamaParameters = struct { + prefill_tokens: zml.Tensor, + decode_tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: llama.KvCache, + rng: zml.Tensor.Rng, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, +}; + +const CompileModelResult = struct { + embedding_prefill_exe: zml.Exe, + embedding_decode_exe: zml.Exe, + layer_prefill_exe: zml.Exe, + layer_decode_exe: zml.Exe, + norm_prefill_exe: zml.Exe, + norm_decode_exe: zml.Exe, + sampling_prefill_exe: zml.Exe, + sampling_decode_exe: zml.Exe, +}; + +pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: llama.LlamaLM, + parameters: LlamaParameters, + progress: *std.Progress.Node, +) !CompileModelResult { + _ = progress; // autofix + var timer = try std.time.Timer.start(); + log.info("Compiling model components", .{}); + defer log.info("Compiled all components [{D}]", .{timer.read()}); + + const dtype = llama_model.model.embed_tokens.weight.dtype(); + const hidden_dim = llama_model.config.hidden_size; + + // Build hidden states tensors for prefill and decode compilation + const hidden_shape_prefill = zml.Shape.init(.{ .s = parameters.prefill_tokens.dim(.s), .d = hidden_dim }, dtype); + const hidden_states_prefill = zml.Tensor.init(hidden_shape_prefill, dtype); + + const hidden_shape_decode = zml.Shape.init(.{ .s = parameters.decode_tokens.dim(.s), .d = hidden_dim }, dtype); + const hidden_states_decode = zml.Tensor.init(hidden_shape_decode, dtype); + + // embedding + var embed_prefill_future = io.async(struct { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: zml.nn.TokenEmbedding, input: zml.Tensor) !zml.Exe { + return plt.compile(alloc, io_, mod, .forward, .{input}); + } + }.call, .{ allocator, io, platform, llama_model.model.embed_tokens, parameters.prefill_tokens }); + errdefer if (embed_prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + + var embed_decode_future = io.async(struct { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: zml.nn.TokenEmbedding, input: zml.Tensor) !zml.Exe { + return plt.compile(alloc, io_, mod, .forward, .{input}); + } + }.call, .{ allocator, io, platform, llama_model.model.embed_tokens, parameters.decode_tokens }); + errdefer if (embed_decode_future.cancel(io)) |v| v.deinit() else |_| {}; + + // layer + const layer_module = llama_model.model.layers[0]; + + var layer_prefill_future = io.async(struct { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: llama.TransformerLayer, hidden: zml.Tensor, idx: zml.Tensor, kv: llama.KvCache, param: LlamaParameters) !zml.Exe { + return plt.compile(alloc, io_, mod, .forward, .{ hidden, idx, kv, param.attention_metadata, param.attention_parameters }); + } + }.call, .{ allocator, io, platform, layer_module, hidden_states_prefill, parameters.token_index, parameters.kv_cache, parameters }); + errdefer if (layer_prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + + var layer_decode_future = io.async(struct { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: llama.TransformerLayer, hidden: zml.Tensor, idx: zml.Tensor, kv: llama.KvCache, param: LlamaParameters) !zml.Exe { + return plt.compile(alloc, io_, mod, .forward, .{ hidden, idx, kv, param.attention_metadata, param.attention_parameters }); + } + }.call, .{ allocator, io, platform, layer_module, hidden_states_decode, parameters.token_index, parameters.kv_cache, parameters }); + errdefer if (layer_decode_future.cancel(io)) |v| v.deinit() else |_| {}; + + // norm + var norm_prefill_future = io.async(struct { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: llama.RmsNorm, hidden: zml.Tensor) !zml.Exe { + return plt.compile(alloc, io_, mod, .forward, .{hidden}); + } + }.call, .{ allocator, io, platform, llama_model.model.norm, hidden_states_prefill }); + errdefer if (norm_prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + + var norm_decode_future = io.async(struct { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: llama.RmsNorm, hidden: zml.Tensor) !zml.Exe { + return plt.compile(alloc, io_, mod, .forward, .{hidden}); + } + }.call, .{ allocator, io, platform, llama_model.model.norm, hidden_states_decode }); + errdefer if (norm_decode_future.cancel(io)) |v| v.deinit() else |_| {}; + + // sampling + var sampling_prefill_future = io.async(struct { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, model: llama.LlamaLM, head: zml.nn.Linear, hidden: zml.Tensor, rng: zml.Tensor.Rng, opts: zml.nn.SamplingStrategy) !zml.Exe { + return plt.compile(alloc, io_, model, .sampleTokens, .{ head, hidden, rng, opts }); + } + }.call, .{ allocator, io, platform, llama_model, llama_model.lm_head.?, hidden_states_prefill, parameters.rng, llama_model.gen_opts }); + errdefer if (sampling_prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + + var sampling_decode_future = io.async(struct { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, model: llama.LlamaLM, head: zml.nn.Linear, hidden: zml.Tensor, rng: zml.Tensor.Rng, opts: zml.nn.SamplingStrategy) !zml.Exe { + return plt.compile(alloc, io_, model, .sampleTokens, .{ head, hidden, rng, opts }); + } + }.call, .{ allocator, io, platform, llama_model, llama_model.lm_head.?, hidden_states_decode, parameters.rng, llama_model.gen_opts }); + errdefer if (sampling_decode_future.cancel(io)) |v| v.deinit() else |_| {}; + + // Wait for execs compiled + const embedding_prefill_exe = try embed_prefill_future.await(io); + const embedding_decode_exe = try embed_decode_future.await(io); + const layer_prefill_exe = try layer_prefill_future.await(io); + const layer_decode_exe = try layer_decode_future.await(io); + const norm_prefill_exe = try norm_prefill_future.await(io); + const norm_decode_exe = try norm_decode_future.await(io); + const sampling_prefill_exe = try sampling_prefill_future.await(io); + const sampling_decode_exe = try sampling_decode_future.await(io); + + return .{ + .embedding_prefill_exe = embedding_prefill_exe, + .embedding_decode_exe = embedding_decode_exe, + .layer_prefill_exe = layer_prefill_exe, + .layer_decode_exe = layer_decode_exe, + .norm_prefill_exe = norm_prefill_exe, + .norm_decode_exe = norm_decode_exe, + .sampling_prefill_exe = sampling_prefill_exe, + .sampling_decode_exe = sampling_decode_exe, + }; +} + +fn loadModelBuffers( + allocator: std.mem.Allocator, + buffers_allocator: std.mem.Allocator, + io: std.Io, + progress: *std.Progress.Node, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + llama_model: llama.LlamaLM, + pool: *zml.mem.DynamicBufferPool, + async_limit: usize, +) !zml.Bufferized(llama.LlamaLM) { + var transferred_bytes: usize = 0; + + var timer = try stdx.time.Timer.start(); + defer { + const duration = timer.read(); + const seconds = @as(f64, @floatFromInt(duration.ns)) / 1e9; + const gb_per_sec = @as(f64, @floatFromInt(transferred_bytes)) / (1024.0 * 1024.0 * 1024.0) / seconds; + const gbps = gb_per_sec * 8.0; + log.info("Loaded model [{D} {d:.3} GB/s {d:.3} gbps]", .{ duration, gb_per_sec, gbps }); + } + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + var group: stdx.Io.LimitedGroup = .init(async_limit); + defer group.cancel(io); + + const bufferize_ctx: zml.io.BufferizeContext(llama.TransferCtx) = .{ + .allocator = allocator, + .arena = &arena, + .io = io, + .platform = platform, + .cb_ctx = .{ + .allocator = buffers_allocator, + .pool = pool, + .transferred_bytes = &transferred_bytes, + .progress = progress, + }, + }; + + const bufferized = llama_model.loadBuffers(bufferize_ctx, &group, store.view(), transferBuffer); + try group.await(io); + + return bufferized; +} + +fn transferBuffer(ctx: zml.io.TensorBufferTransfer(llama.TransferCtx)) !void { + var transfer_progress = ctx.cb_ctx.progress.start(ctx.tensor.name, ctx.tensor.byteSize()); + + var reader = zml.safetensors.TensorReader.init(ctx.io, ctx.tensor, &.{}) catch unreachable; + defer reader.deinit(); + + var writer = zml.io.DeviceWriter2.init(ctx.cb_ctx.allocator, ctx.io, ctx.platform, ctx.cb_ctx.pool, ctx.tensor.shape, ctx.buffer, .device) catch unreachable; + defer { + writer.interface.flush() catch unreachable; + writer.deinit(); + } + + _ = reader.interface.streamRemaining(&writer.interface) catch unreachable; + writer.interface.flush() catch unreachable; + + transfer_progress.end(); + + ctx.cb_ctx.transferred_bytes.* += ctx.tensor.byteSize(); +} + +pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + if (skip_llama3_encoding) { + // Copy so the ownership is the same in both branches. + return try allocator.dupe(u32, try encoder.encode(prompt)); + } + + const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); + + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline }); + + try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); + + return tokens.toOwnedSlice(allocator); +} + +pub fn generateText( + allocator: std.mem.Allocator, + io: std.Io, + llama_buffers: *zml.Bufferized(LlamaLM), + embedding_prefill_exe: zml.Exe, + embedding_decode_exe: zml.Exe, + layer_prefill_exe: zml.Exe, + layer_decode_exe: zml.Exe, + norm_prefill_exe: zml.Exe, + norm_decode_exe: zml.Exe, + sampling_prefill_exe: zml.Exe, + sampling_decode_exe: zml.Exe, + kv_cache_buffers: *zml.Bufferized(llama.KvCache), + attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + tokenizer: zml.tokenizer.Tokenizer, + config: LlamaLM.Config, + options: LlamaLM.Options, + seed: u128, + prompt: []const u8, + skip_llama3_encoding: bool, + writer: *std.Io.Writer, + platform: *const zml.Platform, +) !void { + const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); + defer allocator.free(prompt_tok); + + const hidden_dim = config.hidden_size; + const dtype = .bf16; // todo + + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + const max_seq_len = options.max_seq_len; + + // init RNG and buffers + var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io); + defer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + var generated_token_slice: zml.Slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .i32)); + defer generated_token_slice.free(allocator); + + const KvCacheLayerBuffer = struct { + k: zml.Buffer, + v: zml.Buffer, + layer_index: zml.Buffer, + }; + + const num_layers = config.num_hidden_layers; + var kv_layer_buffers = try allocator.alloc(KvCacheLayerBuffer, num_layers); + defer { + for (kv_layer_buffers) |input| { + var idx = input.layer_index; + idx.deinit(); + } + allocator.free(kv_layer_buffers); + } + for (kv_layer_buffers, 0..) |*kv_layer, j| { + const idx_buf = try zml.Buffer.scalar(io, platform, @as(i32, @intCast(j)), .u32); + kv_layer.* = .{ + .k = kv_cache_buffers.k, + .v = kv_cache_buffers.v, + .layer_index = idx_buf, + }; + } + + // Prepare args and results for decode steps + var embedding_decode_args = try embedding_decode_exe.args(allocator); + defer embedding_decode_args.deinit(allocator); + var embedding_decode_results = try embedding_decode_exe.results(allocator); + defer embedding_decode_results.deinit(allocator); + + var layer_decode_args = try layer_decode_exe.args(allocator); + defer layer_decode_args.deinit(allocator); + var layer_decode_results = try layer_decode_exe.results(allocator); + defer layer_decode_results.deinit(allocator); + + var norm_decode_args = try norm_decode_exe.args(allocator); + defer norm_decode_args.deinit(allocator); + var norm_decode_results = try norm_decode_exe.results(allocator); + defer norm_decode_results.deinit(allocator); + + var sampling_decode_args = try sampling_decode_exe.args(allocator); + defer sampling_decode_args.deinit(allocator); + var sampling_decode_results = try sampling_decode_exe.results(allocator); + defer sampling_decode_results.deinit(allocator); + + var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); + defer current_token_buffer.deinit(); + + const decode_hidden_shape = zml.Shape.init(.{ .s = 1, .d = hidden_dim }, dtype); + var decode_hidden_buffer = try zml.Buffer.uninitialized(io, platform, decode_hidden_shape, .{}); + defer decode_hidden_buffer.deinit(); + + { + const prefill_tokens_slice: zml.Slice = try .alloc(allocator, .init(.{max_seq_len}, .i32)); + defer prefill_tokens_slice.free(allocator); + + // Secure copy with cast to i32 because sampling output is i32 + @memset(prefill_tokens_slice.items(i32), 0); + const prompt_i32 = std.mem.bytesAsSlice(i32, std.mem.sliceAsBytes(prompt_tok)); + @memcpy(prefill_tokens_slice.items(i32)[0..prompt_tok.len], prompt_i32); + + // Prepare buffers + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); + defer prefill_tokens_buffer.deinit(); + var prefill_token_pos_buffer = try zml.Buffer.scalar(io, platform, 0, .u32); + defer prefill_token_pos_buffer.deinit(); + + const hidden_shape = zml.Shape.init(.{ .s = max_seq_len, .d = hidden_dim }, dtype); + var prefill_hidden_buffer = try zml.Buffer.uninitialized(io, platform, hidden_shape, .{}); + defer prefill_hidden_buffer.deinit(); + + // Execute embedding, layers, norm and sampling + var embedding_args = try embedding_prefill_exe.args(allocator); + defer embedding_args.deinit(allocator); + var embedding_results = try embedding_prefill_exe.results(allocator); + defer embedding_results.deinit(allocator); + + embedding_args.set(.{ llama_buffers.model.embed_tokens, prefill_tokens_buffer }); + embedding_prefill_exe.call(embedding_args, &embedding_results); + embedding_results.fill(.{&prefill_hidden_buffer}); + + var layer_args = try layer_prefill_exe.args(allocator); + defer layer_args.deinit(allocator); + var layer_results = try layer_prefill_exe.results(allocator); + defer layer_results.deinit(allocator); + + for (llama_buffers.model.layers, 0..) |layer_weights, i| { + kv_layer_buffers[i].k = kv_cache_buffers.k; + kv_layer_buffers[i].v = kv_cache_buffers.v; + layer_args.set(.{ layer_weights, prefill_hidden_buffer, prefill_token_pos_buffer, kv_layer_buffers[i], attention_metadata_buffers }); + layer_prefill_exe.call(layer_args, &layer_results); + layer_results.fill(.{ &prefill_hidden_buffer, kv_cache_buffers }); + } + + var norm_args = try norm_prefill_exe.args(allocator); + defer norm_args.deinit(allocator); + var norm_results = try norm_prefill_exe.results(allocator); + defer norm_results.deinit(allocator); + + norm_args.set(.{ llama_buffers.model.norm, prefill_hidden_buffer }); + norm_prefill_exe.call(norm_args, &norm_results); + norm_results.fill(.{&prefill_hidden_buffer}); + + var sampling_args = try sampling_prefill_exe.args(allocator); + defer sampling_args.deinit(allocator); + var sampling_results = try sampling_prefill_exe.results(allocator); + defer sampling_results.deinit(allocator); + + var full_sequence_output_buffer = try zml.Buffer.uninitialized(io, platform, zml.Shape.init(.{ .s = max_seq_len }, .u32), .{}); + defer full_sequence_output_buffer.deinit(); + + sampling_args.set(.{ llama_buffers, llama_buffers.lm_head, prefill_hidden_buffer, rng_buffers }); + sampling_prefill_exe.call(sampling_args, &sampling_results); + sampling_results.fill(.{ &full_sequence_output_buffer, &rng_buffers }); + + try full_sequence_output_buffer.toSlice(io, prefill_tokens_slice); + + generated_token_slice.items(i32)[0] = prefill_tokens_slice.items(i32)[prompt_tok.len - 1]; + } + + const output_tokens_len = max_seq_len - prompt_tok.len - 1; + var timer = try stdx.time.Timer.start(); + var num_tokens_generated: usize = 1; + + // Decode + generation: for (0..output_tokens_len + 1) |i| { + num_tokens_generated += 1; + const generated_token = generated_token_slice.items(u32)[0]; + + if (try tokenizer_decoder.next(generated_token)) |chunk| { + try writer.writeAll(chunk); + try writer.flush(); + } + + if (i == output_tokens_len) break :generation; + switch (config.eos_token_id.value) { + .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, + .ints => |eos_list| { + for (eos_list) |eos| { + if (generated_token == @as(u32, @intCast(eos))) break :generation; + } + }, + } + + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(prompt_tok.len + i)})); + var token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice); + defer token_pos_buffer.deinit(); + + embedding_decode_args.set(.{ llama_buffers.model.embed_tokens, current_token_buffer }); + embedding_decode_exe.call(embedding_decode_args, &embedding_decode_results); + embedding_decode_results.fill(.{&decode_hidden_buffer}); + + for (llama_buffers.model.layers, 0..) |layer_weights, j| { + kv_layer_buffers[j].k = kv_cache_buffers.k; + kv_layer_buffers[j].v = kv_cache_buffers.v; + layer_decode_args.set(.{ layer_weights, decode_hidden_buffer, token_pos_buffer, kv_layer_buffers[j], attention_metadata_buffers }); + layer_decode_exe.call(layer_decode_args, &layer_decode_results); + layer_decode_results.fill(.{ &decode_hidden_buffer, kv_cache_buffers }); + } + + norm_decode_args.set(.{ llama_buffers.model.norm, decode_hidden_buffer }); + norm_decode_exe.call(norm_decode_args, &norm_decode_results); + norm_decode_results.fill(.{&decode_hidden_buffer}); + + sampling_decode_args.set(.{ llama_buffers, llama_buffers.lm_head, decode_hidden_buffer, rng_buffers }); + sampling_decode_exe.call(sampling_decode_args, &sampling_decode_results); + sampling_decode_results.fill(.{ ¤t_token_buffer, &rng_buffers }); + + try current_token_buffer.toSlice(io, generated_token_slice); + } + + const duration = timer.read(); + std.debug.print("\n", .{}); + log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ num_tokens_generated, duration, duration.div(num_tokens_generated).hzFloat() }); +} From eb05f39fb34a9a4675ac06fa0d0b816ad10c5bd6 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 12 Feb 2026 20:43:00 +0100 Subject: [PATCH 5/6] WIP --- examples/llama/llama.zig | 6 +++--- examples/llama/main_layers.zig | 30 ++++++++++++++++++++++++------ platforms/metal/BUILD.bazel | 26 +++++++++++++++++++++++--- platforms/metal/metal.zig | 19 +++++++++++++++++-- zml/buffer.zig | 13 +++++++------ zml/pjrtx.zig | 14 ++++++++------ 6 files changed, 82 insertions(+), 26 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 2d9268eb..3d41f32c 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -119,8 +119,8 @@ pub const LlamaLM = struct { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache, attention_metadata, attention_parameters); - const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } pub fn sampleTokens( @@ -293,7 +293,7 @@ pub const TransformerLayer = struct { } }; -const RmsNorm = struct { +pub const RmsNorm = struct { weight: Tensor, eps: f32 = 1e-5, diff --git a/examples/llama/main_layers.zig b/examples/llama/main_layers.zig index 38f704f7..70f45c3b 100644 --- a/examples/llama/main_layers.zig +++ b/examples/llama/main_layers.zig @@ -61,7 +61,9 @@ pub fn main() !void { // defer if (debug_allocator) |*da| std.debug.assert(da.deinit() == .ok); const allocator = std.heap.c_allocator; - var threaded: std.Io.Threaded = .init(allocator, .{}); + var threaded: std.Io.Threaded = .init(allocator, .{ + .async_limit = .limited(1), + }); defer threaded.deinit(); const args = stdx.flags.parseProcessArgs(CliArgs); @@ -108,7 +110,7 @@ pub fn main() !void { const llama_options: llama.LlamaLM.Options = .{ .max_seq_len = args.seqlen, .sampling_strategy = .{ - .topk = 2, + .topk = 1, .temperature = 1.0, }, }; @@ -325,6 +327,8 @@ pub fn compile( // embedding var embed_prefill_future = io.async(struct { fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: zml.nn.TokenEmbedding, input: zml.Tensor) !zml.Exe { + log.info("Compiling prefill", .{}); + defer log.info("Compiled prefill", .{}); return plt.compile(alloc, io_, mod, .forward, .{input}); } }.call, .{ allocator, io, platform, llama_model.model.embed_tokens, parameters.prefill_tokens }); @@ -332,6 +336,8 @@ pub fn compile( var embed_decode_future = io.async(struct { fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: zml.nn.TokenEmbedding, input: zml.Tensor) !zml.Exe { + log.info("Compiling embed", .{}); + defer log.info("Compiled embed", .{}); return plt.compile(alloc, io_, mod, .forward, .{input}); } }.call, .{ allocator, io, platform, llama_model.model.embed_tokens, parameters.decode_tokens }); @@ -342,6 +348,8 @@ pub fn compile( var layer_prefill_future = io.async(struct { fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: llama.TransformerLayer, hidden: zml.Tensor, idx: zml.Tensor, kv: llama.KvCache, param: LlamaParameters) !zml.Exe { + log.info("Compiling layer prefill", .{}); + defer log.info("Compiled layer prefill", .{}); return plt.compile(alloc, io_, mod, .forward, .{ hidden, idx, kv, param.attention_metadata, param.attention_parameters }); } }.call, .{ allocator, io, platform, layer_module, hidden_states_prefill, parameters.token_index, parameters.kv_cache, parameters }); @@ -349,6 +357,8 @@ pub fn compile( var layer_decode_future = io.async(struct { fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: llama.TransformerLayer, hidden: zml.Tensor, idx: zml.Tensor, kv: llama.KvCache, param: LlamaParameters) !zml.Exe { + log.info("Compiling layer decode", .{}); + defer log.info("Compiled layer decode", .{}); return plt.compile(alloc, io_, mod, .forward, .{ hidden, idx, kv, param.attention_metadata, param.attention_parameters }); } }.call, .{ allocator, io, platform, layer_module, hidden_states_decode, parameters.token_index, parameters.kv_cache, parameters }); @@ -357,6 +367,8 @@ pub fn compile( // norm var norm_prefill_future = io.async(struct { fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: llama.RmsNorm, hidden: zml.Tensor) !zml.Exe { + log.info("Compiling norm prefill", .{}); + defer log.info("Compiled norm prefill", .{}); return plt.compile(alloc, io_, mod, .forward, .{hidden}); } }.call, .{ allocator, io, platform, llama_model.model.norm, hidden_states_prefill }); @@ -364,6 +376,8 @@ pub fn compile( var norm_decode_future = io.async(struct { fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, mod: llama.RmsNorm, hidden: zml.Tensor) !zml.Exe { + log.info("Compiling norm decode", .{}); + defer log.info("Compiled norm decode", .{}); return plt.compile(alloc, io_, mod, .forward, .{hidden}); } }.call, .{ allocator, io, platform, llama_model.model.norm, hidden_states_decode }); @@ -371,17 +385,21 @@ pub fn compile( // sampling var sampling_prefill_future = io.async(struct { - fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, model: llama.LlamaLM, head: zml.nn.Linear, hidden: zml.Tensor, rng: zml.Tensor.Rng, opts: zml.nn.SamplingStrategy) !zml.Exe { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, model: llama.LlamaLM, head: ?zml.nn.Linear, hidden: zml.Tensor, rng: zml.Tensor.Rng, opts: zml.nn.SamplingStrategy) !zml.Exe { + log.info("Compiling sampling prefill", .{}); + defer log.info("Compiled sampling prefill", .{}); return plt.compile(alloc, io_, model, .sampleTokens, .{ head, hidden, rng, opts }); } - }.call, .{ allocator, io, platform, llama_model, llama_model.lm_head.?, hidden_states_prefill, parameters.rng, llama_model.gen_opts }); + }.call, .{ allocator, io, platform, llama_model, llama_model.lm_head, hidden_states_prefill, parameters.rng, llama_model.gen_opts }); errdefer if (sampling_prefill_future.cancel(io)) |v| v.deinit() else |_| {}; var sampling_decode_future = io.async(struct { - fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, model: llama.LlamaLM, head: zml.nn.Linear, hidden: zml.Tensor, rng: zml.Tensor.Rng, opts: zml.nn.SamplingStrategy) !zml.Exe { + fn call(alloc: std.mem.Allocator, io_: std.Io, plt: *const zml.Platform, model: llama.LlamaLM, head: ?zml.nn.Linear, hidden: zml.Tensor, rng: zml.Tensor.Rng, opts: zml.nn.SamplingStrategy) !zml.Exe { + log.info("Compiling sampling decode", .{}); + defer log.info("Compiled sampling decode", .{}); return plt.compile(alloc, io_, model, .sampleTokens, .{ head, hidden, rng, opts }); } - }.call, .{ allocator, io, platform, llama_model, llama_model.lm_head.?, hidden_states_decode, parameters.rng, llama_model.gen_opts }); + }.call, .{ allocator, io, platform, llama_model, llama_model.lm_head, hidden_states_decode, parameters.rng, llama_model.gen_opts }); errdefer if (sampling_decode_future.cancel(io)) |v| v.deinit() else |_| {}; // Wait for execs compiled diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel index aa2c0c6a..8a731e83 100644 --- a/platforms/metal/BUILD.bazel +++ b/platforms/metal/BUILD.bazel @@ -1,6 +1,6 @@ load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") cc_library( name = "empty", @@ -10,7 +10,7 @@ cc_library( copy_to_directory( name = "sandbox", srcs = [ - "libIREECompiler.so", + "libIREECompiler.dylib", "pjrt_plugin_iree_metal.dylib", "@toolchains_llvm_bootstrapped//tools:lld", ], @@ -18,7 +18,7 @@ copy_to_directory( include_external_repositories = ["**"], replace_prefixes = { "pjrt_plugin_iree_metal.dylib": "lib/pjrt_plugin_iree_metal.dylib", - "libIREECompiler.so": "lib/libIREECompiler.so", + "libIREECompiler.dylib": "lib/libIREECompiler.dylib", }, ) @@ -45,3 +45,23 @@ zig_library( "//conditions:default": [":empty"], }), ) + +cc_library( + name = "smi_h", + hdrs = ["smi.h"], +) + +zig_binary( + name = "smi", + linkopts = [ + "-framework", + "CoreFoundation", + "-framework", + "IOKit", + ], + main = "device.zig", + visibility = ["//visibility:public"], + deps = [ + ":smi_h", + ], +) diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig index 67732b63..63fde303 100644 --- a/platforms/metal/metal.zig +++ b/platforms/metal/metal.zig @@ -18,7 +18,7 @@ fn setupEnv(sandbox_path: []const u8) !void { var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; _ = c.setenv( "IREE_PJRT_COMPILER_LIB_PATH", - @ptrCast(try stdx.Io.Dir.path.bufJoinZ(&path_buf, &.{ sandbox_path, "lib/libIREECompiler.so" })), + @ptrCast(try stdx.Io.Dir.path.bufJoinZ(&path_buf, &.{ sandbox_path, "lib/libIREECompiler.dylib" })), 1, ); // must be zero terminated _ = c.setenv( @@ -26,6 +26,11 @@ fn setupEnv(sandbox_path: []const u8) !void { @ptrCast(try stdx.Io.Dir.path.bufJoinZ(&path_buf, &.{ sandbox_path, "bin/lld" })), 1, ); + _ = c.setenv( + "IREE_PJRT_IREE_COMPILER_OPTIONS", + "--iree-opt-level=O3", + 1, + ); } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { @@ -53,6 +58,16 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "pjrt_plugin_iree_metal.dylib" }); - break :blk pjrt.Api.loadFrom(path); + + var lib: std.DynLib = .{ + .inner = .{ + .handle = std.c.dlopen(path, .{ .LAZY = true, .LOCAL = true }) orelse { + log.err("Unable to dlopen plugin: {s}", .{path}); + return error.FileNotFound; + }, + }, + }; + + break :blk pjrt.Api.fromDynLib(&lib); }; } diff --git a/zml/buffer.zig b/zml/buffer.zig index 0a83bcb6..098b5c5d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -138,10 +138,9 @@ pub const Buffer = struct { pub fn uninitialized(io: std.Io, platform: *const Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { if (platform.target == .metal) { - const slice: Slice = try .alloc(std.heap.c_allocator, shape_); - defer slice.free(std.heap.c_allocator); - - return .fromSliceOpts(io, platform, slice, .{}); + const slice: Slice = try .alloc(std.heap.c_allocator, shape_); + defer slice.free(std.heap.c_allocator); + return .fromSlice(io, platform, slice); } var res: Buffer = .{ .platform = platform, @@ -216,11 +215,13 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the provided slice. pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { + _ = io; // autofix //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self.platform.pjrt_api, slice.data()); if (maybe_event) |event| { - try event.await(self.platform.pjrt_api, io); - event.deinit(self.platform.pjrt_api); + _ = event; // autofix + // try event.await(self.platform.pjrt_api, io); + // event.deinit(self.platform.pjrt_api); } } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 6cb4ca3b..20c9d75a 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -22,12 +22,14 @@ pub const Client = opaque { }; }; - const stablehlo_version = blk: { - if (api.stablehloCurrentVersion()) |requested_version| { - break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); - } - break :blk dialects.stablehlo.minimumVersion(); - }; + // const stablehlo_version = blk: { + // if (api.stablehloCurrentVersion()) |requested_version| { + // break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); + // } + // std.log.info(">>>>>>> {s}", .{dialects.stablehlo.minimumVersion()}); + // break :blk dialects.stablehlo.minimumVersion(); + // }; + const stablehlo_version = "1.13.0"; var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer serialized_buffer.deinit(); From 0695720918f30b7562bda58a43284083a7fd0cf5 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 12 Feb 2026 21:00:05 +0100 Subject: [PATCH 6/6] pjrt: add from DynLib --- pjrt/pjrt.zig | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 8a213ffa..b7cda789 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -116,8 +116,13 @@ pub const Api = struct { return err; }, }; + + return fromDynLib(&lib); + } + + pub fn fromDynLib(lib: *std.DynLib) !*const Api { const DynGetPjrtApi = lib.lookup(*const fn () callconv(.c) *const Api, "GetPjrtApi") orelse { - std.debug.panic("Unable to find GetPjrtApi symbol in library: {s}", .{library}); + std.debug.panic("Unable to find GetPjrtApi symbol in library", .{}); }; const api = DynGetPjrtApi();