From 0ca0f44aa8992c4878dd9bb72c1b0ce86bdaac7a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 29 Sep 2025 14:46:53 +0200 Subject: [PATCH 1/5] add patches to http_deb_archive --- bazel/http_deb_archive.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/http_deb_archive.bzl b/bazel/http_deb_archive.bzl index f847e6f3..91b52f5c 100644 --- a/bazel/http_deb_archive.bzl +++ b/bazel/http_deb_archive.bzl @@ -39,6 +39,7 @@ http_deb_archive = repository_rule( "strip_prefix": attr.string(), "build_file": attr.label(allow_single_file = True), "build_file_content": attr.string(), + "patches": attr.label_list(allow_files = True), "workspace_file": attr.label(allow_single_file = True), "workspace_file_content": attr.string(), }, From f5c0bf8d094642cc48cc19c873477fa2f0b2d4ef Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 3 Oct 2025 14:26:56 +0200 Subject: [PATCH 2/5] add documentation for command_buffer_compatible --- zml/callback.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zml/callback.zig b/zml/callback.zig index 5af96d15..ea7a4b23 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -120,7 +120,13 @@ pub fn call( pub const Config = struct { output_operand_aliases: []const i64 = &.{}, copy_inputs_to_host_pinned: bool = false, - // TODO: document precisely what `command_buffer_compatible` is doing and its limitations. + + /// Indicates that the handler is compatible with command buffers (ie Cuda graphs). + /// The Cuda backend traces the execution of the callback and records it as a Cuda graph. + /// To be compatible with CUDA graphs, the callback need: + /// 1. to not use any synchronous operations + /// 2. to not allocate any new host/device memory + /// See: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#prohibited-and-unhandled-operations. traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, // TODO: handle sharded inputs has_side_effect: bool = true, From b03b8f93aaed415da0719d5791c0b0b33b7a3527 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 6 Oct 2025 11:32:44 +0200 Subject: [PATCH 3/5] pinToCore --- stdx/stdx.zig | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 46983493..3ec8b8bb 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -24,3 +24,13 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { } pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; + +pub fn pinToCore(core_id: usize) void { + if (builtin.os.tag == .linux) { + const CPUSet = std.bit_set.ArrayBitSet(usize, std.os.linux.CPU_SETSIZE * @sizeOf(usize)); + + var set: CPUSet = .initEmpty(); + set.set(core_id); + std.os.linux.sched_setaffinity(0, @ptrCast(&set.masks)) catch {}; + } +} From 323be7bfb852f247cd742fb6465675c5837ac97c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 6 Oct 2025 16:28:22 +0200 Subject: [PATCH 4/5] describeLayout --- zml/meta.zig | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/zml/meta.zig b/zml/meta.zig index 891a27ec..6be95b31 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -172,7 +172,6 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } if (@sizeOf(ToStruct) == 0) return; - switch (type_info_to) { .@"struct" => |info| inline for (info.fields) |field| { if (field.is_comptime or @sizeOf(field.type) == 0) continue; @@ -193,7 +192,17 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } else { stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); }, - else => @field(to, field.name) = @field(from, field.name), + else => { + if (field.type == To) { + if (@FieldType(FromStruct, field.name) == To) { + @field(to, field.name) = @field(from, field.name); + } else { + @field(to, field.name) = cb(ctx, @field(from, field.name)); + } + } else { + @field(to, field.name) = @field(from, field.name); + } + }, } }, .@"union" => { @@ -774,3 +783,73 @@ pub fn Contains(Haystack: type, T: type) bool { else => false, }; } + +pub const Handle = enum(u32) { _ }; + +pub fn describeLayout( + T: type, + allocator: std.mem.Allocator, + model: anytype, +) std.mem.Allocator.Error!struct { + *MapRestrict(T, Handle).map(@TypeOf(model)), + []Handle, +} { + const t_count: usize = count(T, &model); + // Assume that we have at worse one slice per item. + // If this become an issue, we could write a custom `count` function that computes the exact size needed, accounting for slices. + const pessimist_size = t_count * (@sizeOf([]Handle) / @sizeOf(Handle) + @sizeOf(Handle)); + const layout_buffer = try allocator.alloc(Handle, pessimist_size); + errdefer allocator.free(layout_buffer); + + var fba: std.heap.FixedBufferAllocator = .init(@ptrCast(layout_buffer)); + var counter: u32 = 0; + const res = try fba.allocator().create(MapRestrict(T, Handle).map(@TypeOf(model))); + try mapAlloc(struct { + fn cb(counter_: *u32, _: T) Handle { + defer counter_.* += 1; + return @enumFromInt(counter_.*); + } + }.cb, fba.allocator(), &counter, model, res); + + std.debug.assert(counter == t_count); + const used_handles = @divExact(std.mem.alignForward(usize, fba.end_index, @alignOf(Handle)), @sizeOf(Handle)); + const owned_handles = if (allocator.resize(layout_buffer, used_handles)) used_handles else pessimist_size; + return .{ res, layout_buffer[0..owned_handles] }; +} + +test describeLayout { + const A = struct { value: f32 = 0.0 }; + const AA = struct { a: A, aa: ?A }; + const AModel = struct { + top: A, + metadata: u32, + layers: []const AA, + bottom: A, + }; + + { + const model: AModel = .{ + .top = .{}, + .metadata = 0xff, + .layers = &.{ .{ .a = .{}, .aa = null }, .{ .a = .{}, .aa = .{} }, .{ .a = .{}, .aa = null } }, + .bottom = .{}, + }; + const ALayout = MapRestrict(A, Handle).map(AModel); + const layout: *const ALayout, const layout_mem: []Handle = try describeLayout(A, std.testing.allocator, model); + defer std.testing.allocator.free(layout_mem); + + try std.testing.expectEqualDeep(&ALayout{ + .top = @enumFromInt(0), + .layers = &.{ + .{ .a = @enumFromInt(1) }, + .{ .a = @enumFromInt(2), .aa = @enumFromInt(3) }, + .{ .a = @enumFromInt(4) }, + }, + .bottom = @enumFromInt(5), + }, layout); + try std.testing.expectEqual( + @intFromPtr(layout) + @sizeOf(ALayout), + @intFromPtr(layout.layers.ptr), + ); + } +} From 3bec54182336c5941baa615072ee683f0ed530e6 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 7 Oct 2025 11:03:03 +0200 Subject: [PATCH 5/5] flat Bufferized struct --- examples/llama/llama.zig | 14 ++++---- examples/llama/main.zig | 55 ++++++++++++++++++++++------- zml/aio.zig | 13 +++++-- zml/buffer.zig | 4 ++- zml/exe.zig | 5 +-- zml/meta.zig | 24 ++++++++++--- zml/ops.zig | 31 ++++++++++++++-- zml/tensor.zig | 76 +++++++++++++++++++++++++++++++++++++--- 8 files changed, 185 insertions(+), 37 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 5d0e87d5..7882d7b2 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -339,12 +339,14 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { - return .{ - .k = try zml.Buffer.uninitialized(platform, kv_shape, .{}), - .v = try zml.Buffer.uninitialized(platform, kv_shape, .{}), - .layer_index = try zml.Buffer.scalar(platform, 0, .u32), - }; + pub fn initBuffer(allocator: std.mem.Allocator, kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { + var kv: zml.Bufferized(KvCache) = try .initStatic(allocator); + kv.set(kv.handles.k, try zml.Buffer.uninitialized(platform, kv_shape, .{})); + kv.set(kv.handles.v, try zml.Buffer.uninitialized(platform, kv_shape, .{})); + kv.set(kv.handles.layer_index, try zml.Buffer.scalar(platform, 0, .u32)); + + log.warn("kv: {f}", .{kv}); + return kv; } pub fn keys(self: KvCache) Tensor { diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6d577695..b67e5ced 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -83,25 +83,36 @@ pub fn generateText( const max_seq_len = llama_.model.max_seq_len; // init RNG and buffers - var rng = try zml.Tensor.Rng.init(platform, seed); + const rng = try zml.Tensor.Rng.init(allocator, platform, seed); + // defer rng.deinit(allocator); var generated_token_buffer = [_]u32{undefined}; - var kv_cache = prefill: { + var prefill_res = prefill: { // prepare device buffers for the prefill tokens and their positions const prefill_buffer = try allocator.alloc(u32, max_seq_len); @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); - var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); - defer prefill_tokens.deinit(); - var prefill_token_pos = try zml.Buffer.scalar(platform, 0, .u32); - defer prefill_token_pos.deinit(); + const PrefillArgs = stdx.meta.FnParam(@TypeOf(mod_prefill).call, 1); + const args: PrefillArgs = try .initStatic(allocator); - const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); - _ = try prefilled_tokens.toHost(std.mem.sliceAsBytes(prefill_buffer)); + const prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); + args.set(args.handles[0], prefill_tokens); + + const prefill_token_pos = try zml.Buffer.scalar(platform, 0, .u32); + defer prefill_token_pos.deinit(); + args.set(args.handles[1], prefill_token_pos); + + args.setSlice(args.handles[2].k, kv_cache_.buffers); + args.setSlice(args.handles[3]._state, rng.buffers); + + // std.log.warn("prefill_args: {f}", .{args}); + const prefill_res = mod_prefill.call(args); + // std.log.warn("prefill_res: {f}", .{prefill_res}); + _ = try prefill_res.buffers[0].toHost(std.mem.sliceAsBytes(prefill_buffer)); generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; - break :prefill kv_cache; + break :prefill prefill_res; }; - defer zml.aio.unloadBuffers(&kv_cache); + // defer prefill_res.deinit(std.heap.smp_allocator); // Prepare for token-by-token generation, // start with the token generated based on the full prompt. @@ -114,6 +125,14 @@ pub fn generateText( // One token has alreadyh been generated by the prefill. var num_tokens_generated: usize = 1; + const GenArgs = stdx.meta.FnParam(@TypeOf(mod_generate).call, 1); + const args: GenArgs = try .initStatic(allocator); + // defer args.deinit(allocator); + + // args 0,1 are set in the loop + args.setSlice(args.handles[2].k, prefill_res.getSlice(prefill_res.handles[1].k, zml.ops.staticCountTensors(llama.KvCache).?)); + args.set(args.handles[3]._state, prefill_res.get(prefill_res.handles[2]._state)); + generation: for (0..output_tokens_len + 1) |i| { // collect and print generated sequence num_tokens_generated += 1; @@ -138,9 +157,20 @@ pub fn generateText( const token_pos = try zml.Buffer.fromSlice(platform, .{}, token_pos_buffer); defer token_pos.deinit(); + args.set(args.handles[0], current_token); + args.set(args.handles[1], token_pos); // call to generate the next token - current_token, kv_cache, rng = mod_generate.call(.{ current_token, token_pos, kv_cache, rng }); + // std.log.warn("gen_args: {f}", .{args}); + const gen_res = mod_generate.call(args); + // Note: we should pass gen_res to call so it's clear who owns what. + defer gen_res.deinitStorage(std.heap.smp_allocator); + // std.log.warn("gen_res: {f}", .{gen_res}); + // Donation invalidates the pjrt.Buffer handles, so we need to copy this manually. + // It's kinda annoying. Could we do that as part of `call` ? + current_token = gen_res.get(gen_res.handles[0]); + args.setSlice(args.handles[2].k, gen_res.getField(KvCache, gen_res.handles[1])); + args.setSlice(args.handles[3]._state, gen_res.getField(Tensor.Rng, gen_res.handles[2])); // extract the generated token from the buffer _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); } @@ -310,7 +340,8 @@ pub fn asyncMain() !void { defer llama_module.deinit(); log.info("✅\tCompiled model in {D}", .{start.read()}); log.info("Creating KvCache", .{}); - const kv_cache = try llama.KvCache.initBuffer(kv_shape, platform); + const kv_cache = try llama.KvCache.initBuffer(allocator, kv_shape, platform); + // defer kv_cache.deinit(allocator); var tokenizer = blk: { log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); diff --git a/zml/aio.zig b/zml/aio.zig index 8114e0a1..698fb15c 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -146,9 +146,16 @@ pub const BufferStore = struct { } }; - var res: zml.Bufferized(Model) = undefined; - try zml.meta.mapAlloc(Ctx.cb, allocator, .{ .platform = &platform, .store = &self }, model, &res); - return res; + const layout = try zml.meta.describeLayout(zml.Tensor, allocator, model); + errdefer layout.deinit(allocator); + const buffers = try allocator.alloc(zml.Buffer, layout.count); + + zml.meta.collectBuf(Ctx.cb, .{ .platform = &platform, .store = &self }, &model, buffers); + return .{ + .buffers = buffers, + .handles = layout.layout, + .handles_memory = layout.layout_memory, + }; } pub fn getTensor(self: BufferStore, key: []const u8) zml.Tensor { diff --git a/zml/buffer.zig b/zml/buffer.zig index a4a9b194..0ac2ed1a 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -327,6 +327,7 @@ pub const Buffer = struct { /// but just marked as available in the memory pool. pub fn deinit(self: *const Buffer) void { // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); + // log.warn("Deinit {f}", .{self}); for (self._shards.constSlice()) |buffer| { buffer.deinit(self._api); } @@ -362,7 +363,8 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try writer.print("Buffer({f})@{x}", .{ self._shape, self.devicePtr() }); + const device_ptr: ?*anyopaque = self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch null; + try writer.print("0x{x}@{f}", .{ @intFromPtr(device_ptr), self._shape }); } pub fn getMemory(self: Buffer) *const pjrt.Memory { diff --git a/zml/exe.zig b/zml/exe.zig index 15d33b0c..4fc3bab1 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -392,8 +392,9 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { const total_ready = fillBuffers(&args, self.inner.input_shapes, self.inner.input_per_device, self.inner.ready_buffer_count); std.debug.assert(total_ready == self.inner.input_buffer_count); self.inner._unsafeCall(); - var result: Bufferized(ReturnT) = undefined; - self.inner._unsafeAssignResults(Bufferized(ReturnT), &result); + const result = Bufferized(ReturnT).initStatic(std.heap.smp_allocator) catch @panic("TODO: pass allocator to call"); + var flat: struct { []Buffer } = .{result.buffers}; + self.inner._unsafeAssignResults(@TypeOf(flat), &flat); return result; } diff --git a/zml/meta.zig b/zml/meta.zig index 6be95b31..2eb2ddf1 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -760,6 +760,7 @@ pub fn Contains(Haystack: type, T: type) bool { anyopaque => return false, else => {}, } + // @compileLog(Haystack, T); return switch (@typeInfo(Haystack)) { .@"struct" => |info| { @@ -786,14 +787,23 @@ pub fn Contains(Haystack: type, T: type) bool { pub const Handle = enum(u32) { _ }; +pub fn Layout(ModelT: type, T: type) type { + return struct { + layout: *MapRestrict(T, Handle).map(ModelT), + layout_memory: []Handle, + count: u32, + + pub fn deinit(self: @This(), allocator: std.mem.Allocator) void { + return allocator.free(self.layout_memory); + } + }; +} + pub fn describeLayout( T: type, allocator: std.mem.Allocator, model: anytype, -) std.mem.Allocator.Error!struct { - *MapRestrict(T, Handle).map(@TypeOf(model)), - []Handle, -} { +) std.mem.Allocator.Error!Layout(@TypeOf(model), T) { const t_count: usize = count(T, &model); // Assume that we have at worse one slice per item. // If this become an issue, we could write a custom `count` function that computes the exact size needed, accounting for slices. @@ -814,7 +824,11 @@ pub fn describeLayout( std.debug.assert(counter == t_count); const used_handles = @divExact(std.mem.alignForward(usize, fba.end_index, @alignOf(Handle)), @sizeOf(Handle)); const owned_handles = if (allocator.resize(layout_buffer, used_handles)) used_handles else pessimist_size; - return .{ res, layout_buffer[0..owned_handles] }; + return .{ + .layout = res, + .layout_memory = layout_buffer[0..owned_handles], + .count = counter, + }; } test describeLayout { diff --git a/zml/ops.zig b/zml/ops.zig index edbcd4a2..d7ecb76d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -723,16 +723,41 @@ pub fn staticCountTensors(comptime T: type) ?usize { return n; }, .@"struct" => |struct_info| { - var count: usize = 0; - inline for (struct_info.fields) |field| { - count += staticCountTensors(field.type) orelse return null; + comptime var count: usize = 0; + comptime { + for (struct_info.fields) |field| { + count += staticCountTensors(field.type) orelse return null; + } } return count; }, + .optional => |opt_info| { + const n = staticCountTensors(opt_info.child) orelse return null; + if (n > 0) return null; + }, else => 0, }; } +test staticCountTensors { + try std.testing.expectEqual(1, staticCountTensors(Tensor)); + try std.testing.expectEqual(8, staticCountTensors([8]Tensor)); + try std.testing.expectEqual(4, staticCountTensors(struct { + a: Tensor, + b: struct { ba: Tensor, bb: Tensor, bc: Tensor }, + })); + + try std.testing.expectEqual(10, staticCountTensors(struct { + a: Tensor, + b: [3]struct { ba: Tensor, bb: Tensor, bc: Tensor }, + })); + + try std.testing.expectEqual(null, staticCountTensors(struct { + a: Tensor, + b: struct { ba: Tensor, bb: ?Tensor, bc: Tensor }, + })); +} + /// Create a Tensor struct similar to base, keeping base tags, /// but using mlir value and dims from the mlir operation. pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base) { diff --git a/zml/tensor.zig b/zml/tensor.zig index d5acc541..676cad33 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -523,10 +523,11 @@ pub const Tensor = struct { }; } - pub fn init(platform: Platform, seed: u128) !Bufferized(Rng) { - return .{ - ._state = try Buffer.fromBytes(platform, Rng.shape()._state, std.mem.asBytes(&seed)), - }; + pub fn init(allocator: std.mem.Allocator, platform: Platform, seed: u128) !Bufferized(Rng) { + const rng: Bufferized(Rng) = try .initStatic(allocator); + rng.set(rng.handles._state, try Buffer.fromBytes(platform, Rng.shape()._state, std.mem.asBytes(&seed))); + + return rng; } /// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. @@ -4082,7 +4083,72 @@ test "Tensor.maxPool2d" { /// Non-Tensor metadata is stripped out of the resulting struct. /// Recursively descends into the type. pub fn Bufferized(comptime T: type) type { - return meta.MapRestrict(Tensor, Buffer).map(T); + return struct { + buffers: []Buffer, + handles: *const meta.MapRestrict(Tensor, meta.Handle).map(T), + handles_memory: []const meta.Handle, + + pub fn initStatic(allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + const template: T = undefined; + const N = comptime ops.staticCountTensors(T) orelse stdx.debug.compileError("Bufferized(T).initStatic is only available for struct with a compile-time known number of Buffers. Got {}", .{T}); + const buffers = try allocator.alloc(Buffer, N); + errdefer allocator.free(buffers); + const layout = try meta.describeLayout(Tensor, allocator, template); + std.debug.assert(layout.count == N); + return .{ + .buffers = buffers, // uninitialized + .handles = layout.layout, + .handles_memory = layout.layout_memory, + }; + } + + pub fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("[buffers: {f}, handles: {any}]", .{ stdx.fmt.slice(self.buffers), self.handles }); + } + + pub fn get(self: @This(), handle: meta.Handle) Buffer { + return self.buffers[@intFromEnum(handle)]; + } + + pub fn set(self: @This(), handle: meta.Handle, x: Buffer) void { + self.buffers[@intFromEnum(handle)] = x; + } + + pub fn getSlice(self: @This(), handle: meta.Handle, n: usize) []const Buffer { + return self.buffers[@intFromEnum(handle)..][0..n]; + } + + pub fn setSlice(self: @This(), handle: meta.Handle, others: []const Buffer) void { + return @memcpy(self.buffers[@intFromEnum(handle)..][0..others.len], others); + } + + pub fn getField(self: @This(), FieldType: type, handle: meta.MapRestrict(Tensor, meta.Handle).map(FieldType)) []const Buffer { + const N = comptime ops.staticCountTensors(FieldType) orelse stdx.debug.compileError("getField(FieldType) is only available for struct with a compile-time known number of Buffers. Got {}", .{FieldType}); + + const first_handle_ptr: *const meta.Handle = @ptrCast(@alignCast(&handle)); + return self.buffers[@intFromEnum(first_handle_ptr.*)..][0..N]; + } + + pub fn setField(self: @This(), FieldType: type, handle: meta.MapRestrict(Tensor, meta.Handle).map(FieldType), buffers: []const Buffer) void { + const N = comptime ops.staticCountTensors(FieldType) orelse stdx.debug.compileError("setField(FieldType) is only available for struct with a compile-time known number of Buffers. Got {}", .{FieldType}); + std.debug.assert(N == buffers.len); + // This is safe cause FieldType has a static number of Tensor so the Handle struct is just N Handle one after the other. + const first_handle_ptr: *const meta.Handle = @ptrCast(@alignCast(&handle)); + return @memcpy(self.buffers[@intFromEnum(first_handle_ptr.*)..][0..N], buffers); + } + + pub fn deinit(self: @This(), allocator: std.mem.Allocator) void { + for (self.buffers) |buff| { + buff.deinit(); + } + self.deinitStorage(allocator); + } + + pub fn deinitStorage(self: @This(), allocator: std.mem.Allocator) void { + allocator.free(self.buffers); + allocator.free(self.handles_memory); + } + }; } /// Return a clone of a type with Tensors replaced by Shapes.