From d1d4e75412e2b17d6941f0d1de363703d5500d6f Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:21:50 +0100 Subject: [PATCH 001/103] stdx: fix Duration.hz and delete the dead AllocatingLimitedConcurrentGroup Duration.hz divides the i96 nanosecond count with `/`, which does not compile for signed integers once the function is called; it now uses @divTrunc, with a test that calls it. AllocatingLimitedConcurrentGroup has no user in zml or llmd and is deleted. LimitedGroup stays: zml/io.zig's Loader uses it. Co-Authored-By: Claude Opus 5.5 --- stdx/Io.zig | 62 +++++------------------------------------------------ 1 file changed, 5 insertions(+), 57 deletions(-) diff --git a/stdx/Io.zig b/stdx/Io.zig index 5b5a35d8..f1ae8669 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -80,72 +80,20 @@ pub const LimitedGroup = struct { } }; -pub const AllocatingLimitedConcurrentGroup = struct { - allocator: std.mem.Allocator, - - queue_buffer: []bool = &[_]bool{}, - limit: usize, - - group: std.Io.Group = .init, - queue: std.Io.Queue(bool) = undefined, - - pub fn init(allocator: std.mem.Allocator, limit: usize) std.mem.Allocator.Error!AllocatingLimitedConcurrentGroup { - const buffer = try allocator.alloc(bool, limit); - @memset(buffer, false); - var self: AllocatingLimitedConcurrentGroup = .{ - .allocator = allocator, - .queue_buffer = buffer, - .limit = limit, - .group = .init, - .queue = undefined, - }; - self.queue = std.Io.Queue(bool).init(self.queue_buffer); - return self; - } - - pub fn deinit(self: *AllocatingLimitedConcurrentGroup) void { - self.allocator.free(self.queue_buffer); - } - - pub fn concurrent( - self: *AllocatingLimitedConcurrentGroup, - io: std.Io, - comptime function: anytype, - args: std.meta.ArgsTuple(@TypeOf(function)), - ) !void { - self.queue.putOneUncancelable(io, true) catch unreachable; - - try self.group.concurrent(io, struct { - fn wrapper( - io_: std.Io, - queue: *std.Io.Queue(bool), - inner_args: std.meta.ArgsTuple(@TypeOf(function)), - ) !void { - defer _ = queue.getOneUncancelable(io_) catch unreachable; - return @call(.auto, function, inner_args); - } - }.wrapper, .{ io, &self.queue, args }); - } - - pub fn await(self: *AllocatingLimitedConcurrentGroup, io: std.Io) std.Io.Cancelable!void { - try self.group.await(io); - } - - pub fn cancel(self: *AllocatingLimitedConcurrentGroup, io: std.Io) void { - self.group.cancel(io); - } -}; - pub const Duration = struct { pub fn div(self: std.Io.Duration, rhs: u64) std.Io.Duration { return .fromNanoseconds(@divTrunc(self.nanoseconds, @as(i96, @intCast(rhs)))); } pub fn hz(self: std.Io.Duration) u64 { - return @intCast(std.time.ns_per_s / self.nanoseconds); + return @intCast(@divTrunc(std.time.ns_per_s, self.nanoseconds)); } pub fn hzFloat(self: std.Io.Duration) f64 { return (1 * std.time.ns_per_s) / @as(f64, @floatFromInt(self.nanoseconds)); } }; + +test "Duration.hz" { + try std.testing.expectEqual(@as(u64, 4), Duration.hz(.fromMilliseconds(250))); +} From f59c2c1fae14b6694684867f72696235e54b1b94 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:29:03 +0100 Subject: [PATCH 002/103] zml/tensor: honor Slice .len before the default .end Slice.absolute tested `.end == to_the_end` before `.len`. Since `.end` defaults to `to_the_end`, a slice given as `.{ .start, .len }` took the rest of the axis instead of `len` elements, contrary to the documented examples. `.len` is now checked first, and the slice test covers it. Co-Authored-By: Claude Opus 5.5 --- zml/tensor.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index 8c148867..b04c5188 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2181,14 +2181,15 @@ pub const Tensor = struct { pub fn absolute(s: Slice, d: i64) Slice { const start = if (s.start < 0) d + s.start else s.start; + // `len` is checked before `end`, whose default would otherwise take the rest of the axis. const end = if (s.singleton) start + 1 + else if (s.len != to_the_end) + start + s.len else if (s.end == to_the_end) d else if (s.end < 0) d + s.end - else if (s.len != to_the_end) - start + s.len else s.end; const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; @@ -2339,6 +2340,7 @@ pub const Tensor = struct { .{ 0, Tensor.Slice{ .end = 1 }, [5]f32{ 0, 1, 2, 3, 4 } }, .{ 1, Tensor.Slice{ .start = 1, .step = 2 }, [4]f32{ 1, 3, 6, 8 } }, .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, + .{ 1, Tensor.Slice{ .start = 1, .len = 2 }, [4]f32{ 1, 2, 6, 7 } }, }) |testcase| { const ax, const slice_, const expectation = testcase; var exe = try zml.module.compile( From 9bdd619e02af37ae36c72310422d55c484f541e3 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:29:12 +0100 Subject: [PATCH 003/103] zml/meta: refuse bounded arrays in MapRestrict `visit` walks only a bounded array's first `len` elements, but the type MapRestrict maps it to holds the whole backing buffer. Pairing a model's tensors with its buffers by visit order would then miss or misplace elements, so mapping a bounded array that contains the source type is now a compile error that points to slices and arrays instead. Co-Authored-By: Claude Opus 5.5 --- zml/meta.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zml/meta.zig b/zml/meta.zig index 1eefea11..504b2973 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -332,6 +332,13 @@ pub fn MapRestrict(From: type, To: type) type { if (!Contains(T, From)) return void; + // `visit` walks a bounded array's `len` elements, but its mapped + // struct would hold the whole buffer: pairing the two by visit + // order would miss or misplace elements. + if (@typeInfo(T) == .@"struct" and @hasDecl(T, "constSlice") and @hasDecl(T, "slice")) { + stdx.debug.compileError("zml.meta.MapRestrict({any}, {any}) cannot map the bounded array {any}: use a slice or an array", .{ From, To, T }); + } + return switch (@typeInfo(T)) { .@"struct" => |struct_infos| { // We know that at least one of the struct field contains a From. From 9c9f601bf04e943bdd394111646c2094a8874a77 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:29:23 +0100 Subject: [PATCH 004/103] zml/platform: refuse more than MAX_NUM_DEVICES devices and free the client on error Platform.init warned when a plugin reported more than MAX_NUM_DEVICES devices, then used them all, although per-device state such as Exe's events lives in arrays of MAX_NUM_DEVICES entries. validateDeviceCount now logs and returns InvalidDeviceCount instead. Platform.init also leaked the PJRT client, and on CUDA the FlashInfer state, when it failed after creating them. Both are now released on error. Co-Authored-By: Claude Opus 5.5 --- zml/platform.zig | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/zml/platform.zig b/zml/platform.zig index 3b52c08e..25f809ed 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -37,6 +37,10 @@ fn validateDeviceCount(target: Target, num_devices: usize) !void { log.err("The selected platform requires at least 1 device, got {}", .{num_devices}); return error.MissingDevices; } + if (num_devices > Platform.MAX_NUM_DEVICES) { + log.err("Platform {} got {} devices, but ZML supports up to {}", .{ target, num_devices, Platform.MAX_NUM_DEVICES }); + return error.InvalidDeviceCount; + } switch (target) { .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi, .furiosa => { if (!std.math.isPowerOfTwo(num_devices)) { @@ -313,11 +317,9 @@ pub const Platform = struct { var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); + errdefer pjrt_client.deinit(api); const pjrt_devices = pjrt_client.addressableDevices(api); try validateDeviceCount(target, pjrt_devices.len); - if (pjrt_devices.len > MAX_NUM_DEVICES) { - log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); - } const pjrt_memories = pjrt_client.addressableMemories(api); @@ -351,6 +353,7 @@ pub const Platform = struct { const arena = platform.arena.allocator(); errdefer platform.arena.deinit(); + errdefer if (comptime Target.cuda.isEnabled()) platform.state.deinit(); try platform.meshes.ensureTotalCapacity(arena, 8); { From a3c9389c94103d58eccfc727517091ae665c0546 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:29:48 +0100 Subject: [PATCH 005/103] zml/quantization: delete the unused quantizeMxfp8 quantizeMxfp8 has no caller in zml or llmd. It also kept its own copy of the platform's FP8 encoding switch, which quantizeInput already holds. Co-Authored-By: Claude Opus 5.5 --- zml/quantization.zig | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/zml/quantization.zig b/zml/quantization.zig index 89e1b210..09688314 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -164,23 +164,6 @@ pub fn quantizeMxfp4(x: Tensor, axis: anytype) QuantizedInput { }; } -pub fn quantizeMxfp8(x: Tensor, axis: anytype) QuantizedInput { - const platform = zml.Compiler.current().platform; - - const dtype: ?DataType = switch (platform.target) { - .cuda, .cpu, .tpu => .f8e4m3fn, - .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => .f8e4m3fn, - .cdna3 => .f8e4m3fnuz, - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, - } else null, - else => null, - }; - - if (dtype == null) std.debug.panic("MXFP8 is not supported on {}", .{platform.target}); - return quantizeBlockFp8(x, axis, mx_block_size, dtype.?, .f8e8m0); -} - fn ceilPowerOfTwo(raw_scale: Tensor) Tensor { const bits = raw_scale.bitCast(.u32); return bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); From 16ba006f5437866da89364468ed4cbcf632f42b2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:29:55 +0100 Subject: [PATCH 006/103] zml/testing: fix expectClose's doc and delete BufferizedWithArgs expectClose's doc names zml.HostBuffer, which no longer exists; it accepts a zml.Slice. BufferizedWithArgs has no user and is deleted. Buffer's doc likewise pointed to zml.aio.loadBuffers and HostBuffer.toDevice, neither of which exists; it now names fromSlice and fromBytes. Co-Authored-By: Claude Opus 5.5 --- zml/buffer.zig | 3 +-- zml/testing.zig | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 8714d80f..9da62c68 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -27,8 +27,7 @@ test { /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. /// /// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. -/// * loading weights from disk directly to the `device zml.aio.loadBuffers` -/// * can be created by calling `HostBuffer.toDevice(platform)`. +/// * created from host data by `fromSlice` or `fromBytes`. pub const Buffer = struct { _platform: *const Platform, _shape: Shape, diff --git a/zml/testing.zig b/zml/testing.zig index 5e3d6333..853f7d2e 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -57,8 +57,8 @@ fn toSlice(allocator: std.mem.Allocator, io: std.Io, slice_or_buffer: anytype) ! } else .{ slice_or_buffer, false }; } -/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the -/// host for comparison ! +/// Testing utility. Accepts both zml.Buffer and zml.Slice; a zml.Buffer is copied to the host for +/// comparison. pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpts) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; var left: Slice, const should_free_left = try toSlice(allocator, io, left_); @@ -365,10 +365,6 @@ pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpec return error.TestExpectedEqual; } -fn BufferizedWithArgs(comptime T: type) type { - return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); -} - fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { var count: usize = 0; var it = store.store.registry.iterator(); From 6061902c648891f62dca6401e9d50c4c693f0a71 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:32:31 +0100 Subject: [PATCH 007/103] examples, docs: fix stale snippets and delete dead example loaders Docs that no longer match the code: - huggingface_access_token.md names HF_TOKEN, the variable vfs/hf.zig reads, instead of HUGGINGFACE_TOKEN. - write_first_model.md calls Buffer.fromSlice with its three arguments and explains that compile() reads each tensor's sharding from its shape, instead of a `sharding` variable the tutorial never defines. - howto_torch2zml.md calls Platform.deinit with its io, and replaces maybeCreateReplicatedTensor, which does not exist, with maybeCreateTensor(..., .replicated, .{}). In examples/llm, lfm2, llama, qwen3_5 and qwen3_5_moe each kept a second loader on their inner Model (loadBuffers or load) that nothing calls and that would not compile if called; they are deleted. LoadedModel.loadBuffers stays the only loader. qwen3_5_moe binds its .replicated tensors with an empty partitioning, which the store ignores for the replicated mesh anyway, and examples/io iterates the registry's tensors directly. Neither changes behavior. Co-Authored-By: Claude Opus 5.5 --- docs/howtos/howto_torch2zml.md | 8 ++--- docs/howtos/huggingface_access_token.md | 4 +-- docs/tutorials/write_first_model.md | 18 +++++----- examples/io/main.zig | 4 +-- examples/llm/models/lfm2/model.zig | 32 ------------------ examples/llm/models/llama/model.zig | 32 ------------------ examples/llm/models/qwen3_5/model.zig | 33 ------------------ examples/llm/models/qwen3_5_moe/model.zig | 41 +++-------------------- 8 files changed, 21 insertions(+), 151 deletions(-) diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 3d246f11..31584751 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -206,17 +206,17 @@ pub fn main(init: std.process.Init) !void { const mlp: Mlp = .{ .up_proj = .init( mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), + mlp_view.maybeCreateTensor("up_proj.bias", .{.dout}, .replicated, .{}), .d, ), .gate_proj = .init( mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), + mlp_view.maybeCreateTensor("gate_proj.bias", .{.dout}, .replicated, .{}), .d, ), .down_proj = .init( mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), - mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), + mlp_view.maybeCreateTensor("down_proj.bias", .{.d}, .replicated, .{}), .dout, ), }; @@ -242,7 +242,7 @@ pub fn main(init: std.process.Init) !void { ... var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); var mlp_weights = try zml.io.load(Mlp, &mlp, allocator, io, platform, &model_store, .auto); } ``` diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index cc11ffbb..9f18b4a7 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -17,7 +17,7 @@ access token to log into your account on Huggingface, directly from `bazel`, in * the token looks something like `hf_abCdEfGhijKlM`. * store the token on your machine (replace the placeholder with your actual token): -You can use the `HUGGINGFACE_TOKEN` environment variable to store the token or use +You can use the `HF_TOKEN` environment variable to store the token or use its standard location: ``` mkdir -p $HOME/.cache/huggingface/; echo > "$HOME/.cache/huggingface/token" @@ -29,6 +29,6 @@ Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! ``` # requires token in $HOME/.cache/huggingface/token, as created by the -# `hf auth login` command, or the `HUGGINGFACE_TOKEN` environment variable. +# `hf auth login` command, or the `HF_TOKEN` environment variable. bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 312c9afc..e9f13eaa 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -175,9 +175,9 @@ var executable = try platform.compile(allocator, io, layer, .forward, .{input}, defer executable.deinit(); ``` -You might wonder what this `sharding` variable is for ? ZML supports sharding tensors across multiple devices, -and the `compile()` function needs to know how the tensors are sharded in order to compile the module correctly. -For this simple example, we just replicate the tensors across all devices for simplicity. +ZML supports sharding tensors across multiple devices, and `compile()` reads how +each tensor is sharded from its shape. A tensor created from a shape alone, as +here, is replicated on every device. ### Creating the "bufferized" Model @@ -191,8 +191,8 @@ provides ways to have extra-low loading times). const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 2.0, 3.0})); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 1.0, 1.0})); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); @@ -213,7 +213,7 @@ To create the _Args_ and _Results_, the _Executable_ exposes two conveniences. ```zig // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); -var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); +var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); // create the Args and Results structs @@ -372,15 +372,15 @@ pub fn main(init: std.process.Init) !void { const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 2.0, 3.0 })); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 1.0, 1.0 })); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); // create the Args and Results structs diff --git a/examples/io/main.zig b/examples/io/main.zig index 1cec23b0..396cfdfd 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -132,7 +132,7 @@ pub fn main(init: std.process.Init) !void { const root = try TensorNode.init(init.arena.allocator(), ""); - var registry_it = registry.iterator(); + var registry_it = registry.tensors.iterator(); while (registry_it.next()) |kv| { const name = kv.key_ptr.*; const tensor = kv.value_ptr.*; @@ -183,7 +183,7 @@ pub fn main(init: std.process.Init) !void { const tensors = try allocator.alloc(zml.Tensor, tensor_count); defer allocator.free(tensors); - var registry_it = registry.iterator(); + var registry_it = registry.tensors.iterator(); var load_count: usize = 0; while (registry_it.next()) |entry| : (load_count += 1) { tensors[load_count] = switch (mesh_type) { diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index dd21c7f9..a769d599 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -164,38 +164,6 @@ pub const Model = struct { }; } - pub fn loadBuffers( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 8, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { self.embed_tokens.weight.deinit(); LmHead.unloadBuffers(&self.lm_head); diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 05610dc1..ee84ea79 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -161,38 +161,6 @@ pub const Model = struct { self.model.deinit(allocator); } - pub fn loadBuffers( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { if (self.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); Llama.unloadBuffers(&self.model, allocator); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 77d3884e..30b29f6a 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -172,39 +172,6 @@ pub const Model = struct { self.text_model.deinit(allocator); } - pub fn load( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - meshes: []const *const zml.Mesh, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store, meshes); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { TextModel.unloadBuffers(&self.text_model, allocator); zml.nn.Linear.unloadBuffers(&self.lm_head); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 11ce8151..84e6db0b 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -188,39 +188,6 @@ pub const Model = struct { self.text_model.deinit(allocator); } - pub fn load( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - meshes: []const *const zml.Mesh, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store, meshes); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { TextModel.unloadBuffers(&self.text_model, allocator); } @@ -709,8 +676,8 @@ const Router = struct { pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { return .{ .router = .init( - store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), - store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), + store.createTensor("weight", .{ .expert, .d }, .replicated, .{}), + store.maybeCreateTensor("bias", .{.expert}, .replicated, .{}), .d, ), .num_experts_per_tok = num_experts_per_tok, @@ -756,8 +723,8 @@ pub const Moe = struct { return .{ .shared_expert = Mlp.init(store.withPrefix("shared_expert")), .shared_expert_gate = .init( - store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), - store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{}), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{}), .d, ), .gate_up_proj = .init(gate_up_proj_tensor, null, .d), From 89f2b044be72d08512c0a4c4f2c190e8be8c5fa2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:34:45 +0100 Subject: [PATCH 008/103] vfs: give standard streams their own descriptors and take a scheme only from scheme:// The VFS seeded its handle table with the cwd placeholder at index 0, then stdin, stdout and stderr, so std.Io.File.stdout() (handle 1) resolved to descriptor 0. Writing to stdout through the VFS went to stdin instead, which only works when both are the same terminal. The standard streams now take indices 0 to 2 and the cwd placeholder follows them. A local name with a colon, such as notes:old.txt, was parsed as a URI: from the cwd it failed as an unregistered scheme, and stripScheme cut the name after the colon. A path is now remote only when it contains "://", stripScheme removes only up to "://", and dirRealPathFile passes the stripped path to the backend, so a remote real path no longer carries its scheme twice. Co-Authored-By: Claude Opus 5.5 --- vfs/vfs.zig | 83 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/vfs/vfs.zig b/vfs/vfs.zig index fe626ff9..dafffe43 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -11,10 +11,20 @@ pub const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/vfs"); -const CWD_HANDLE: u32 = 0; +/// Indices 0-2 are the standard streams, so `std.Io.File.stdout()` and +/// friends resolve to their own descriptor; the cwd placeholder follows. +const CWD_HANDLE: u32 = 3; const VFS = @This(); -const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; +const Handle = struct { + handle: u32, + backend_idx: ?usize, + flags: std.Io.File.Flags = .{ .nonblocking = false }, + + fn innerFile(self: *const Handle) std.Io.File { + return .{ .handle = @intCast(self.handle), .flags = self.flags }; + } +}; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, @@ -29,10 +39,11 @@ pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { const base = VFSBase.init(base_io); var handles: @FieldType(@This(), "handles") = .{}; - try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); + std.debug.assert(handles.len == CWD_HANDLE + 1); return .{ .allocator = allocator, @@ -149,19 +160,8 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz } if (std.meta.eql(dir, std.Io.Dir.cwd())) { - if (sub_path == null) return .{ null, dir, self.base.inner }; - if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; - - const uri = std.Uri.parse(sub_path.?) catch null; - if (uri) |u| { - const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { - if (std.mem.eql(u8, u.scheme, s)) break idx; - } else null; - if (backend_idx == null) return error.VFSNotRegistered; - return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; - } else { - return .{ null, std.Io.Dir.cwd(), self.base.inner }; - } + // Without "://", a path from cwd is local, a colon in it included. + return .{ null, dir, self.base.inner }; } else { const handle = self.getDirHandle(dir); if (handle.backend_idx) |backend_idx| { @@ -172,9 +172,11 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz } } +/// `path` without its "scheme://" prefix; a name with a colon and no "://" +/// is a plain local name. fn stripScheme(path: []const u8) []const u8 { - const uri = std.Uri.parse(path) catch return path; - return path[uri.scheme.len + 3 ..]; + const i = std.mem.indexOf(u8, path, "://") orelse return path; + return path[i + 3 ..]; } fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { @@ -330,10 +332,10 @@ fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8 if (self.getScheme(backend_idx)) |s| { const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, stripScheme(path_name), out_buffer[prefix.len..]); return prefix.len + path_len; } else { - return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); + return try backend.vtable.dirRealPathFile(backend.userdata, dir_, stripScheme(path_name), out_buffer); } } @@ -437,6 +439,19 @@ fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std. } } +test "standard streams resolve to their own descriptors" { + var vfs: VFS = try .init(std.testing.allocator, std.testing.io); + defer vfs.deinit(); + + const files = [_]std.Io.File{ .stdin(), .stdout(), .stderr() }; + const fds = [_]std.posix.fd_t{ std.posix.STDIN_FILENO, std.posix.STDOUT_FILENO, std.posix.STDERR_FILENO }; + for (files, fds) |file, fd| { + const handle, const backend = vfs.getFileHandle(file); + try std.testing.expectEqual(fd, handle.innerFile().handle); + try std.testing.expect(backend.vtable == vfs.base.inner.vtable); + } +} + test "copy a local file through the VFS" { const allocator = std.testing.allocator; const process_io = std.testing.io; @@ -481,3 +496,31 @@ test "copy a local file through the VFS" { defer allocator.free(copied); try std.testing.expectEqualSlices(u8, &source_data, copied); } + +test "a name with a colon and no \"://\" is local" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "notes:old.txt", .data = "bytes" }); + var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; + + var vfs: VFS = try .init(std.testing.allocator, testing_io); + defer vfs.deinit(); + const vfs_io = vfs.io(); + + const backend_idx, _, _ = try vfs.lookupDir(.cwd(), "notes:old.txt"); + try std.testing.expectEqual(null, backend_idx); + + const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); + defer dir.close(vfs_io); + const file = try dir.openFile(vfs_io, "notes:old.txt", .{}); + defer file.close(vfs_io); + var contents: [5]u8 = undefined; + try std.testing.expectEqual(contents.len, try file.readPositionalAll(vfs_io, &contents, 0)); + try std.testing.expectEqualStrings("bytes", &contents); + + var real_path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const real_path = real_path_buffer[0..try dir.realPathFile(vfs_io, "notes:old.txt", &real_path_buffer)]; + try std.testing.expect(std.mem.endsWith(u8, real_path, "/notes:old.txt")); +} From abbf55593755d7b6de932eac1a947de16de23fa9 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:39:57 +0100 Subject: [PATCH 009/103] vfs/gcs: fix gs:// object paths, fail truncated streaming reads and delete dead GCS code gcsUri set an object's path to the VFS path, bucket/key, without the leading slash, so every GCS request targeted `bucket/key` instead of `/bucket/key`. storage.googleapis.com accepts that, but it is not a valid origin-form target and other endpoints need not. The path is now rooted, with a test, and gcs.zig's tests run under //vfs:test. A streaming read whose request failed returned EndOfStream on http, hf, s3 and gs, so a reader saw a short file instead of an error. It now returns InputOutput; the error is still logged where it happens. The GCS backend also loses code nothing reaches: the JWT assertion and its commented-out signing, getTimestamp, the commented-out authHeaders, EndpointUrl, and the `region` option, which no caller sets. Co-Authored-By: Claude Opus 5.5 --- vfs/gcs.zig | 135 ++++++++------------------------------------------- vfs/hf.zig | 2 +- vfs/http.zig | 2 +- vfs/s3.zig | 2 +- vfs/vfs.zig | 4 ++ 5 files changed, 28 insertions(+), 117 deletions(-) diff --git a/vfs/gcs.zig b/vfs/gcs.zig index acc7f634..8169c2da 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -8,7 +8,6 @@ const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/vfs/gcs"); -const EndpointUrl = "https://storage.googleapis.com"; const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; /// Check if running on Google Compute Engine / Cloud Run host infrastructure, @@ -51,29 +50,11 @@ const Credentials = union(enum) { }; const ServiceAccount = struct { - pub const JwtAssertion = struct { - iss: []const u8, - scope: []const u8 = "https://www.googleapis.com/auth/devstorage.read_only", - aud: []const u8 = "https://oauth2.googleapis.com/token", - exp: i64, - iat: i64, - sub: ?[]const u8 = null, - }; - type: []const u8, client_email: []const u8, private_key: []const u8, token_uri: []const u8, quota_project_id: ?[]const u8 = null, - - pub fn makeAssertion(self: *const ServiceAccount, io_: std.Io) JwtAssertion { - const now: std.Io.Timestamp = .now(io_, .real); - return .{ - .iss = self.client_email, - .iat = now.toSeconds(), - .exp = now.addDuration(.fromSeconds(3600)).toSeconds(), - }; - } }; authorized_user: AuthorizedUser, @@ -274,7 +255,6 @@ pub const GCS = struct { pub const Config = struct { credentials: ?Credentials = null, endpoint_uri: std.Uri, - region: []const u8 = "auto", }; const Handle = struct { @@ -317,7 +297,6 @@ pub const GCS = struct { metadata_server: void, } = null, endpoint_url: []const u8 = "https://storage.googleapis.com", - region: []const u8 = "auto", read_pool: parallel_read.InitOpts = .{ .chunk_size = 16 << 20, .num_workers = 32, @@ -359,7 +338,6 @@ pub const GCS = struct { .metadata_server => .{ .metadata_server = {} }, } else null, .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, opts.endpoint_url)) catch return InitError.Unexpected, - .region = try arena.allocator().dupe(u8, opts.region), }; const token: Token = .{ @@ -469,44 +447,6 @@ pub const GCS = struct { _ = service_account; _ = buffer; return error.Unimplemented; - - // const assertion = service_account.makeAssertion(io_); - - // var tmp_buffer: [8 * 1024]u8 = undefined; - - // var buffer_writer: std.Io.Writer = .fixed(buffer); - // var signing_input_buffer: [2 * 1024]u8 = undefined; - // var signing_input_writer: std.Io.Writer = .fixed(signing_input_buffer); - - // const encoded_rs256_header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; - // try signing_input_writer.print("{s}.", .{encoded_rs256_header}); - // std.base64.url_safe_no_pad.Encoder.encodeWriter( - // signing_input_writer, - // try std.fmt.bufPrint(tmp_buffer, "{f}", .{ - // std.json.fmt(assertion, .{}), - // }), - // ); - // const signing_input = signing_input_writer.buffered(); - // _ = signing_input; - - // const result = try client.fetch(.{ - // .location = .{ .url = "https://oauth2.googleapis.com/token" }, - // .method = .POST, - // .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ - // std.fmt.alt(std.Uri.Component{ .raw = service_account.client_id }, .formatQuery), - // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), - // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), - // }), - // .headers = .{ - // .accept_encoding = .{ .override = "identity" }, - // .content_type = .{ .override = "application/x-www-form-urlencoded" }, - // }, - // .response_writer = &response_writer, - // }); - // if (result.status != .ok) { - // log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); - // return error.RequestFailed; - // } } fn refreshToken(self: *GCS) !void { @@ -642,7 +582,7 @@ pub const GCS = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + return .{ .file_read_streaming = error.InputOutput }; }; if (total == 0) { @@ -864,25 +804,6 @@ pub const GCS = struct { return path.len; } - fn getTimestamp(self: *GCS, buf: []u8) ![]const u8 { - const now_ts = std.Io.Clock.now(.real, self.base.inner); - - const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); - const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; - const year_day = epoch_secs.getEpochDay().calculateYearDay(); - const month_day = year_day.calculateMonthDay(); - const day_seconds = epoch_secs.getDaySeconds(); - - return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ - year_day.year, - @backingInt(month_day.month), - month_day.day_index + 1, - day_seconds.getHoursIntoDay(), - day_seconds.getMinutesIntoHour(), - day_seconds.getSecondsIntoMinute(), - }); - } - fn pathComponents(_: *GCS, path_: []const u8) struct { []const u8, []const u8 } { const path = std.mem.trim(u8, path_, "/"); @@ -893,43 +814,16 @@ pub const GCS = struct { } } - fn gcsUri(self: *GCS, path: []const u8) std.Uri { + /// The object's URI. A VFS path is `bucket/key`; the request target + /// needs `/bucket/key`, formatted into `buffer`, which the URI borrows. + fn gcsUri(self: *GCS, path: []const u8, buffer: []u8) !std.Uri { var uri = self.config.endpoint_uri; - uri.path = .{ .raw = path }; + const absolute = if (std.mem.startsWith(u8, path, "/")) path else std.fmt.bufPrint(buffer, "/{s}", .{path}) catch return error.NameTooLong; + uri.path = .{ .raw = absolute }; uri.query = .{ .percent_encoded = "alt=media" }; return uri; } - // fn authHeaders(self: *GCS, _: std.http.Method, _: std.Uri, _: []const u8, authorization_buffer: []u8) !struct { - // authorization: ?[]const u8, - // extra_headers: [3]?std.http.Header, - // extra_len: usize, - // } { - // if (self.config.oauth_access_token == null) { - // if (self.config.googleCredentialsJsonStr) |json| { - // const oauth = try refreshAuthorizedUserTokenFromJson(self.allocator, self.base.inner, json); - // if (self.config.oauth_access_token) |old_token| self.allocator.free(old_token); - // if (self.config.quota_project_id) |old_project| self.allocator.free(old_project); - // self.config.oauth_access_token = oauth.access_token; - // self.config.quota_project_id = oauth.quota_project_id; - // } - // } - - // const token = self.config.oauth_access_token orelse return error.MissingCredentials; - // const authorization = try std.fmt.bufPrint(authorization_buffer, "Bearer {s}", .{token}); - // var extra_headers: [3]?std.http.Header = .{ null, null, null }; - // var extra_len: usize = 0; - // if (self.config.quota_project_id) |project| { - // extra_headers[0] = .{ .name = "x-goog-user-project", .value = project }; - // extra_len = 1; - // } - // return .{ - // .authorization = authorization, - // .extra_headers = extra_headers, - // .extra_len = extra_len, - // }; - // } - fn listObjectsBody(self: *GCS, bucket: []const u8, key_prefix: []const u8) ![]u8 { var query_buf: [4096]u8 = undefined; var query_writer = std.Io.Writer.fixed(&query_buf); @@ -1033,7 +927,8 @@ pub const GCS = struct { fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { var path_buffer: [8 * 1024]u8 = undefined; const path = try self.resolvePath(dir, sub_path, &path_buffer); - const uri = self.gcsUri(path); + var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; + const uri = try self.gcsUri(path, &uri_path_buffer); var req = try self.client.request(.HEAD, uri, .{ .redirect_behavior = .not_allowed, .headers = .{ @@ -1067,7 +962,8 @@ pub const GCS = struct { const read_size = parallel_read.readSize(handle.size, offset, data); if (read_size == 0) return 0; - const uri = self.gcsUri(handle.uri); + var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; + const uri = try self.gcsUri(handle.uri, &uri_path_buffer); const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); defer self.allocator.free(jobs); @@ -1100,6 +996,17 @@ pub const GCS = struct { } }; +test "GCS object URIs are rooted at the endpoint" { + var backend: GCS = undefined; + backend.config = .{ .endpoint_uri = try std.Uri.parse("https://storage.googleapis.com") }; + + var path_buffer: [64]u8 = undefined; + const uri = try backend.gcsUri("bucket/dir/object.bin", &path_buffer); + var uri_buffer: [128]u8 = undefined; + const formatted = try std.fmt.bufPrint(&uri_buffer, "{f}", .{uri}); + try std.testing.expectEqualStrings("https://storage.googleapis.com/bucket/dir/object.bin?alt=media", formatted); +} + test "GCS parses XML bucket listing objects and common prefixes" { const xml = "" ++ diff --git a/vfs/hf.zig b/vfs/hf.zig index 8770fe18..5e6e767a 100644 --- a/vfs/hf.zig +++ b/vfs/hf.zig @@ -642,7 +642,7 @@ pub const HF = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + return .{ .file_read_streaming = error.InputOutput }; }; if (total == 0) { diff --git a/vfs/http.zig b/vfs/http.zig index 9054983e..a98dc5c6 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -146,7 +146,7 @@ pub const HTTP = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + return .{ .file_read_streaming = error.InputOutput }; }; if (total == 0) { diff --git a/vfs/s3.zig b/vfs/s3.zig index eb849e21..46198ed2 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -534,7 +534,7 @@ pub const S3 = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + return .{ .file_read_streaming = error.InputOutput }; }; if (total == 0) { diff --git a/vfs/vfs.zig b/vfs/vfs.zig index dafffe43..f32cd447 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -9,6 +9,10 @@ pub const HTTP = @import("http.zig").HTTP; pub const S3 = @import("s3.zig").S3; pub const VFSBase = @import("base.zig").VFSBase; +test { + _ = @import("gcs.zig"); +} + const log = std.log.scoped(.@"zml/vfs"); /// Indices 0-2 are the standard streams, so `std.Io.File.stdout()` and From 3c4556709c13a42b4bcf41acc1911b2b0e7906d6 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:43:57 +0100 Subject: [PATCH 010/103] zml/pjrtx: compile on the calling thread pjrtx.Client.compile went through pjrt.Client.compile, which started compileRaw with io.async and awaited it at once. Under zio that ran XLA's compile on an 8 MiB coroutine stack, even though Compiler.compile had already left the coroutine with zio.blockInPlace; this is the likely cause of the stack overflow seen compiling Laguna. pjrtx.Client.compile now calls compileRaw directly and no longer takes an io, Compiler stops passing one, and pjrt.Client.compile, which had no other caller, is deleted. Under threaded io the compile now runs on the caller's thread instead of a pool thread. Under zio, Compiler.compile now leaves the coroutine with zio.blockInPlaceReserved rather than zio.blockInPlace. blockInPlace queues the compile behind the blocking pool's other jobs, and the pool adds a worker only when none is idle and at least twice as many jobs as workers are queued, so a burst of compiles shared one to three threads: llmd's six model compiles took 6 to 15 s under zio on the B70 against 4.5 s under threaded io, and DeepSeek's two on a GB300 ran one after the other (73-77 s against 50 s). A compile holds its thread for its whole run, so each now takes a pool worker of its own, idle or new, and concurrent compiles run in parallel as they do under threaded io. Co-Authored-By: Claude Opus 5.5 --- pjrt/pjrt.zig | 5 ----- zml/Compiler.zig | 8 +++++--- zml/pjrtx.zig | 8 ++++++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 3715361f..e09073b2 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -484,11 +484,6 @@ pub const Client = opaque { return @ptrCast(ret.executable.?); } - pub fn compile(self: *const Client, api: *const Api, io: std.Io, args: CompileArgs) ApiError!*LoadedExecutable { - var future = io.async(compileRaw, .{ self, api, args }); - return future.await(io); - } - pub const BufferFromHostBufferArgs = struct { data: [*]const u8, buffer_type: BufferType, diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 3daba2ec..64dab3ae 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -423,7 +423,10 @@ pub fn compile( ) Error!Exe { return switch (platform.io_impl) { .threaded => compileInternal(allocator, io, platform, func_, args, opts), - .zio => try zio.blockInPlace(struct { + // A compile holds its thread for its whole run, so it takes a pool + // worker of its own instead of queuing behind other jobs; concurrent + // compiles run in parallel. + .zio => try zio.blockInPlaceReserved(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const Platform, args_: @TypeOf(args), opts_: Options) Error!Exe { return try compileInternal(allocator_, io_, platform_, func_, args_, opts_); } @@ -448,7 +451,7 @@ pub fn compileInternal( // in mind we might want to revisit that later. // // This is NOT compatible with std.Io.Evented. - // It is compatible with zio, hence the blockInPlace above. + // It is compatible with zio, hence the blockInPlaceReserved above. const span_name = try tracer.formatSpanName(allocator, "zml.module.compile", .{ .program_name = opts.program_name, @@ -1016,7 +1019,6 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load platform.pjrt_client, platform.pjrt_api, compiler.allocator, - compiler.io, compiler.module, try upb.serialize(options, upb_arena), ); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 824d2bd1..bfa4fc7d 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -14,7 +14,11 @@ const log = std.log.scoped(.@"zml/pjrtx"); pub const Client = opaque { pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || pjrt.ApiError; - pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { + /// Compiles on the calling thread: under zio, `Compiler.compile` has + /// already moved off the coroutine onto a pool worker of its own + /// (`zio.blockInPlaceReserved`), and XLA's compile needs the thread's + /// stack, not a task's. + pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { break :blk stablehlo.smallerVersion(requested_version, stablehlo.currentVersion()); @@ -37,7 +41,7 @@ pub const Client = opaque { }; }; - return @ptrCast(try client.compile(api, io, .{ + return @ptrCast(try client.compileRaw(api, .{ .bytecode = serialized_buffer.written(), .bytecode_format = .mlir, .compile_options_pb = compile_options_pb, From d810269b9b100af9eed028b4c07ea5f058cbf7f3 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:45:39 +0100 Subject: [PATCH 011/103] zml/mem: bufferize array and tuple fields Bufferized maps a [N]Tensor field to [N]Buffer, but bufferize had no case for arrays and reached `unreachable` at runtime on any model with one. It now bufferizes arrays element by element. bufferize also walked Bufferized's field names and looked each one up in the model. Bufferized drops the fields that hold no tensor and, for a tuple, renumbers the ones it keeps, so bufferizing a tuple with a tensor-free field before a tensor did not compile. bufferize now walks the model's fields and maps each to the field Bufferized keeps for it. Co-Authored-By: Claude Opus 5.5 --- zml/mem.zig | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/zml/mem.zig b/zml/mem.zig index a64ae968..b26b1ca1 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -323,6 +323,21 @@ pub fn Bufferized(comptime T: type) type { return meta.MapRestrict(Tensor, Buffer).map(T); } +/// The field of `Bufferized(Model)` that holds the struct `Model`'s field +/// `name`, or null when `Bufferized` strips it. `Bufferized` keeps the +/// fields holding tensors, in order, and renumbers a tuple's. +fn bufferizedField(comptime Model: type, comptime name: []const u8) ?[]const u8 { + @setEvalBranchQuota(10_000); + const info = @typeInfo(Model).@"struct"; + var index: usize = 0; + for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { + if (attrs.@"comptime" or !meta.Contains(Field, Tensor)) continue; + if (std.mem.eql(u8, field_name, name)) return @typeInfo(Bufferized(Model)).@"struct".field_names[index]; + index += 1; + } + return null; +} + fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bufferized(@TypeOf(model))) !void { @setEvalBranchQuota(10_000); const Model = @TypeOf(model); @@ -340,9 +355,11 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const type_info = @typeInfo(ModelBufferized); switch (type_info) { - .@"struct" => |struct_type_info| { - inline for (struct_type_info.field_names) |field_name| { - try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); + .@"struct" => { + inline for (@typeInfo(Model).@"struct".field_names) |field_name| { + if (comptime bufferizedField(Model, field_name)) |buffer_field| { + try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, buffer_field)); + } } }, .@"union" => { @@ -372,6 +389,11 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu else => unreachable, } }, + .array => |info| { + inline for (0..info.len) |index| { + try bufferizeInner(allocator, model[index], &bufferized_.*[index]); + } + }, .void, .int, .@"enum", .bool, .enum_literal, .float, .vector => {}, else => unreachable, } @@ -384,3 +406,18 @@ pub inline fn bufferize(allocator: std.mem.Allocator, comptime ModelType: type, try bufferizeInner(allocator, model.*, &bufferized); return bufferized; } + +test "bufferize maps a tuple's fields as Bufferized renumbers them" { + const Model = struct { u32, Tensor, []Tensor, [2]Tensor }; + const a: Tensor = .init(.{2}, .f32); + const b: Tensor = .init(.{3}, .f16); + var tensors = [_]Tensor{ a, b }; + const model: Model = .{ 3, a, &tensors, .{ b, a } }; + const bufferized = try bufferize(std.testing.allocator, Model, &model); + defer std.testing.allocator.free(bufferized[1]); + try std.testing.expect(bufferized[0].shape().eql(a.shape())); + try std.testing.expectEqual(2, bufferized[1].len); + try std.testing.expect(bufferized[1][1].shape().eql(b.shape())); + try std.testing.expect(bufferized[2][0].shape().eql(b.shape())); + try std.testing.expect(bufferized[2][1].shape().eql(a.shape())); +} From ce116580a5ee6abd001ebef867876936c9983084 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:48:08 +0100 Subject: [PATCH 012/103] zml/mem: add deinitBufferized and roll bufferize back on allocation failure Each model frees its buffers with a hand-written walker that has to follow every field of its Bufferized struct, and leaks when a field is added without updating it. mem.deinitBufferized frees any tree bufferize returns: it deinits every Buffer and frees the slices bufferize allocated, through structs, unions, optionals, arrays and nested slices. Passing it the model instead of its buffers does not compile. When an allocation failed partway through a model, bufferize returned the error and leaked the slices it had already allocated. It now frees what it built before returning the error. It also fills the slice it allocates before storing it, so models with []const Tensor fields bufferize too. Co-Authored-By: Claude Opus 5.5 --- zml/mem.zig | 118 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 6 deletions(-) diff --git a/zml/mem.zig b/zml/mem.zig index b26b1ca1..13d5cb74 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -355,10 +355,16 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const type_info = @typeInfo(ModelBufferized); switch (type_info) { - .@"struct" => { + .@"struct" => |struct_type_info| { + var initialized_fields: usize = 0; + errdefer inline for (struct_type_info.field_names, 0..) |field_name, index| { + if (index < initialized_fields) + deinitBufferized(allocator, &@field(bufferized_, field_name)); + }; inline for (@typeInfo(Model).@"struct".field_names) |field_name| { if (comptime bufferizedField(Model, field_name)) |buffer_field| { try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, buffer_field)); + initialized_fields += 1; } } }, @@ -381,17 +387,27 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); - for (model, bufferized_.*) |src, *dst| { - try bufferizeInner(allocator, src, dst); + const allocated = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); + var initialized: usize = 0; + errdefer { + for (allocated[0..initialized]) |*element| deinitBufferized(allocator, element); + allocator.free(allocated); } + for (model, allocated) |src, *dst| { + try bufferizeInner(allocator, src, dst); + initialized += 1; + } + bufferized_.* = allocated; }, else => unreachable, } }, .array => |info| { + var initialized: usize = 0; + errdefer for (bufferized_.*[0..initialized]) |*element| deinitBufferized(allocator, element); inline for (0..info.len) |index| { try bufferizeInner(allocator, model[index], &bufferized_.*[index]); + initialized = index + 1; } }, .void, .int, .@"enum", .bool, .enum_literal, .float, .vector => {}, @@ -407,17 +423,107 @@ pub inline fn bufferize(allocator: std.mem.Allocator, comptime ModelType: type, return bufferized; } +/// Deinitializes every accelerator buffer of the tree `value` points to and +/// frees the recursive slice storage allocated by `bufferize`. +pub fn deinitBufferized(allocator: std.mem.Allocator, value: anytype) void { + const Ptr = @TypeOf(value); + const T = @typeInfo(Ptr).pointer.child; + if (T == Tensor) @compileError("mem.deinitBufferized takes the buffers bufferize returns, not the model"); + if (T == Buffer) { + const buffer: *Buffer = @constCast(value); + buffer.deinit(); + return; + } + switch (@typeInfo(T)) { + .@"struct" => |info| inline for (info.field_names) |field_name| { + deinitBufferized(allocator, &@field(value, field_name)); + }, + .@"union" => switch (value.*) { + inline else => |*payload| deinitBufferized(allocator, payload), + }, + .optional => if (value.*) |*payload| { + deinitBufferized(allocator, payload); + }, + .pointer => |info| switch (info.size) { + .slice => { + for (value.*) |*element| deinitBufferized(allocator, element); + allocator.free(value.*); + }, + else => @compileError("mem.deinitBufferized takes the buffers bufferize returns, not " ++ @typeName(T)), + }, + .array => for (&value.*) |*element| deinitBufferized(allocator, element), + // What `Bufferized` makes of a union arm or a whole tree without a tensor. + .void => {}, + else => @compileError("mem.deinitBufferized takes the buffers bufferize returns, not " ++ @typeName(T)), + } +} + test "bufferize maps a tuple's fields as Bufferized renumbers them" { const Model = struct { u32, Tensor, []Tensor, [2]Tensor }; const a: Tensor = .init(.{2}, .f32); const b: Tensor = .init(.{3}, .f16); var tensors = [_]Tensor{ a, b }; const model: Model = .{ 3, a, &tensors, .{ b, a } }; - const bufferized = try bufferize(std.testing.allocator, Model, &model); - defer std.testing.allocator.free(bufferized[1]); + var bufferized = try bufferize(std.testing.allocator, Model, &model); + defer deinitBufferized(std.testing.allocator, &bufferized); try std.testing.expect(bufferized[0].shape().eql(a.shape())); try std.testing.expectEqual(2, bufferized[1].len); try std.testing.expect(bufferized[1][1].shape().eql(b.shape())); try std.testing.expect(bufferized[2][0].shape().eql(b.shape())); try std.testing.expect(bufferized[2][1].shape().eql(a.shape())); } + +test "bufferize rolls back earlier slice fields on allocation failure" { + const Model = struct { + first: []const Tensor, + second: []const Tensor, + }; + var tensors: [1]Tensor = undefined; + const model: Model = .{ .first = &tensors, .second = &tensors }; + var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 1 }); + + if (bufferize(failing.allocator(), Model, &model)) |result| { + var unexpected = result; + deinitBufferized(failing.allocator(), &unexpected); + return error.ExpectedOutOfMemory; + } else |err| { + try std.testing.expect(err == error.OutOfMemory); + } + try std.testing.expect(failing.has_induced_failure); + try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); +} + +test "deinitBufferized frees recursive const slices" { + const Layer = struct { weights: []const Tensor }; + const Model = struct { + layers: []const Layer, + fixed: [2]Tensor, + }; + var weights: [2]Tensor = undefined; + const layers = [_]Layer{ + .{ .weights = weights[0..1] }, + .{ .weights = weights[1..2] }, + }; + const model: Model = .{ .layers = &layers, .fixed = undefined }; + var bufferized = try bufferize(std.testing.allocator, Model, &model); + deinitBufferized(std.testing.allocator, &bufferized); +} + +test "deinitBufferized frees slices under optionals and unions" { + const Layer = struct { weights: []const Tensor }; + const Ffn = union(enum) { dense: Layer, experts: []const Layer, none: void }; + const Model = struct { + present: ?Layer, + absent: ?Layer, + ffns: [3]Ffn, + }; + var weights: [3]Tensor = undefined; + const experts = [_]Layer{ .{ .weights = weights[0..1] }, .{ .weights = weights[1..3] } }; + const model: Model = .{ + .present = .{ .weights = &weights }, + .absent = null, + .ffns = .{ .{ .dense = .{ .weights = weights[2..3] } }, .{ .experts = &experts }, .none }, + }; + var bufferized = try bufferize(std.testing.allocator, Model, &model); + deinitBufferized(std.testing.allocator, &bufferized); +} From 59cc71a1b83ce6655ce217b7fe0a673e781c7262 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:49:15 +0100 Subject: [PATCH 013/103] zml/mem: delete the unused FixedBufferPool FixedBufferPool has no user in zml or llmd. Co-Authored-By: Claude Opus 5.5 --- zml/mem.zig | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/zml/mem.zig b/zml/mem.zig index 13d5cb74..44aca8e0 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -179,50 +179,6 @@ pub const DmaMapAllocator = struct { } }; -pub const FixedBufferPool = struct { - buffer: []u8, - block_size: usize, - q_buf: []const u16, - q: std.Io.Queue(u16), - - pub fn init(allocator: std.mem.Allocator, buffer_: []u8, blocks_: u16) !FixedBufferPool { - const block_size = @divExact(buffer_.len, blocks_); - const q_buf = try allocator.alloc(u16, blocks_); - for (q_buf, 0..) |*idx, i| { - idx.* = @intCast(i); - } - var q: std.Io.Queue(u16) = .init(q_buf); - q.type_erased.len = q.type_erased.buffer.len; // make the queue full - return .{ - .buffer = buffer_, - .block_size = block_size, - .q = q, - .q_buf = q_buf, - }; - } - - pub fn deinit(self: *FixedBufferPool, allocator: std.mem.Allocator) void { - allocator.free(self.q_buf); - } - - pub fn get(self: *FixedBufferPool, io: std.Io) ![]u8 { - const idx = try self.q.getOneUncancelable(io); - return self.buffer[idx * self.block_size ..][0..self.block_size]; - } - - fn inRange(sub_buffer: []const u8, buffer: []const u8) bool { - return @intFromPtr(sub_buffer.ptr) >= @intFromPtr(buffer.ptr) and - @intFromPtr(sub_buffer[sub_buffer.len - 1 ..].ptr) <= @intFromPtr(buffer[buffer.len - 1 ..].ptr); - } - - pub fn put(self: *FixedBufferPool, io: std.Io, buf: []u8) void { - // is the pointer in range ? - std.debug.assert(inRange(buf, self.buffer)); - const idx = @divExact(@intFromPtr(buf.ptr) - @intFromPtr(self.buffer.ptr), self.block_size); - self.q.putOneUncancelable(io, @intCast(idx)) catch unreachable; - } -}; - pub const DynamicBufferPool = struct { const Node = struct { next: ?*Node }; const alignment: std.mem.Alignment = .of(Node); From 37ded81affe09d4ef94dc21ca7ce25264e729fc0 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:53:22 +0100 Subject: [PATCH 014/103] zml/exe: return launch errors from Exe.call Exe.call panicked when PJRT failed to launch an executable or when a waited execution event reported an error, so a caller could not recover from, say, a device running out of memory. It now returns pjrt.ApiError. A failed launch writes no outputs. With wait, every partition's event is awaited and destroyed, and a failed execution destroys the outputs it wrote, so the results hold no buffer after any error. The neuron and non-neuron event loops become one; their behavior is unchanged. Runner.run and FnExe's Runner.run keep their panic. Direct callers in the examples, the tutorial and zml.testing add try, and the old Loader.loadExecute reports a failed call as LoadFailed; PJRT has already logged its cause. llmd's one direct call follows in its next zml bump. Co-Authored-By: Claude Opus 5.5 --- docs/tutorials/write_first_model.md | 4 +- examples/benchmark/main.zig | 4 +- examples/llm/models/lfm2_tests.zig | 2 +- examples/llm/models/llama_tests.zig | 2 +- examples/mnist/mnist.zig | 2 +- examples/mnist/mnist_data_parallel.zig | 2 +- examples/sharding/main.zig | 2 +- zml/exe.zig | 54 +++++++++++--------------- zml/io.zig | 2 +- zml/testing.zig | 2 +- 10 files changed, 34 insertions(+), 42 deletions(-) diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index e9f13eaa..69b54590 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -227,7 +227,7 @@ defer results.deinit(allocator); args.set(.{ layer_buffers, input_buffer }); // call our executable -executable.call(io, args, &results, .{}); +try executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); @@ -394,7 +394,7 @@ pub fn main(init: std.process.Init) !void { args.set(.{ layer_buffers, input_buffer }); // call our executable - executable.call(io, args, &results, .{}); + try executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 7ff70707..b78e230b 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -65,14 +65,14 @@ pub fn main(init: std.process.Init) !void { // Ignore first run { - exe.call(io, exe_args, &exe_results, .{}); + try exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); defer result.deinit(); } // call our executable module const run_start: std.Io.Timestamp = .now(io, .awake); - exe.call(io, exe_args, &exe_results, .{}); + try exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); _ = try result.await(io); defer result.deinit(); diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 19b28df6..b9424a04 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -175,7 +175,7 @@ const TestContext = struct { var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - exe.call(self.io, args, &res, .{}); + try exe.call(self.io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 1b5894f1..92828c53 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -126,7 +126,7 @@ fn testLayer( var res = try exe.results(allocator); defer res.deinit(allocator); - exe.call(io, args, &res, .{}); + try exe.call(io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index f5842af8..a6937405 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -147,7 +147,7 @@ pub fn main(init: std.process.Init) !void { printDigit(&sample); args.set(.{ mnist_buffers, input_buffer }); - exe.call(io, args, &results, .{}); + try exe.call(io, args, &results, .{}); var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index a539ae5a..54acc13e 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -176,7 +176,7 @@ pub fn main(init: std.process.Init) !void { args.set(.{ mnist_buffers, input_buffer }); try profiler.start(); - exe.call(io, args, &results, .{}); + try exe.call(io, args, &results, .{}); if (try profiler.stop()) |report| { log.info("Wrote profiler files to {s} and {s}", .{ report.protobuf_path, report.perfetto_path }); } diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index cc46a1bd..29de9597 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -242,7 +242,7 @@ pub fn main(init: std.process.Init) !void { defer exe_results.deinit(allocator); exe_args.set(.{ w_buf, b_buf, input_buf }); - exe.call(io, exe_args, &exe_results, .{}); + try exe.call(io, exe_args, &exe_results, .{}); var out = exe_results.get(zml.Buffer); defer out.deinit(); diff --git a/zml/exe.zig b/zml/exe.zig index 85baa7e7..179df29f 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -310,7 +310,7 @@ pub const Exe = struct { /// Non-donated inputs remain owned by the caller. pub fn run(self: *Runner, io: std.Io, input_values: anytype, output_refs: anytype, opts: CallOpts) void { self.args.set(input_values); - self.exe.call(io, self.args, &self.results, opts); + self.exe.call(io, self.args, &self.results, opts) catch |err| std.debug.panic("PJRT execution failed with: {}", .{err}); self.exe.deinitDonatedInputs(self.args); self.results.fill(output_refs); } @@ -334,7 +334,9 @@ pub const Exe = struct { /// Executes with optional waiting. The caller owns all input handles, including donated inputs. /// Donated input contents cannot be used after the call; output buffers have their own handles. /// Use a runner to destroy donated handles automatically and replace buffers through output destinations. - pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + /// A failed launch writes no outputs; with `wait`, a failed execution destroys the outputs it + /// wrote, so `results_` holds no buffer after any error. + pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) pjrt.ApiError!void { var span = tracer.span("zml.exe.call", .{ .wait = opts.wait, .arg_count = arguments.expected_shapes.len, @@ -342,6 +344,7 @@ pub const Exe = struct { }); defer span.end(); + const api = self.platform.pjrt_api; var events: [Platform.MAX_NUM_DEVICES]?*pjrt.Event = @splat(null); const partition_events = events[0..@intCast(self.num_partitions)]; @@ -350,7 +353,7 @@ pub const Exe = struct { .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) partition_events else null, }; - self.exe.execute(self.platform.pjrt_api, .{ + try self.exe.execute(api, .{ .arguments = arguments.flat_buffers.buffers, .num_args = arguments.expected_shapes.len, .results = results_.flat_buffers.buffers, @@ -358,33 +361,20 @@ pub const Exe = struct { // TODO: Set this flag for undonatable buffers. .non_donatable_input_indices = &.{}, .context = self.context, - }) catch |err| { - std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); - }; + }); - switch (self.platform.target) { - .neuron => { - for (events_slice.?) |e| { - if (e) |ev| { - if (opts.wait) { - ev.await(self.platform.pjrt_api, io) catch |err| { - std.debug.panic("PJRT execution failed with: {}", .{err}); - }; - } - ev.deinit(self.platform.pjrt_api); - } - } - }, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) { - for (events_slice.?) |e| { - if (e) |ev| { - ev.await(self.platform.pjrt_api, io) catch |err| { - std.debug.panic("PJRT execution failed with: {}", .{err}); - }; - ev.deinit(self.platform.pjrt_api); - } - } - }, + // Every partition's event is destroyed, awaited first when waiting. + var failure: ?pjrt.ApiError = null; + if (events_slice) |slice| for (slice) |e| { + const ev = e orelse continue; + if (opts.wait) ev.await(api, io) catch |err| { + failure = failure orelse err; + }; + ev.deinit(api); + }; + if (failure) |err| { + for (results_.flat_buffers.raw_buffers) |buffer| buffer.deinit(api); + return err; } } }; @@ -470,7 +460,7 @@ pub fn FnExe(comptime function_: anytype) type { /// Destroys donated input handles before filling outputs; non-donated inputs remain caller-owned. pub fn run(self: *RunnerSelf, io: std.Io, call: Call, opts: Exe.CallOpts) void { self.args.set(call.inputs); - self.exe.call(io, self.args, &self.results, opts); + self.exe.call(io, self.args, &self.results, opts) catch |err| std.debug.panic("PJRT execution failed with: {}", .{err}); self.exe.deinitDonatedInputs(self.args); self.results.fill(call.outputs); } @@ -620,7 +610,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); args.set(.{ value_buffer, weight_buffer }); - exe.call(io, args, &results, .{ .wait = wait }); + try exe.call(io, args, &results, .{ .wait = wait }); var output = results.get(Buffer); defer output.deinit(); @@ -628,6 +618,8 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { for (value_buffer._shards.constSlice()) |shard| { try std.testing.expect(shard.isDeleted(platform.pjrt_api)); } + // Executing over the deleted input is an error, not a panic. + try std.testing.expect(std.meta.isError(exe.call(io, args, &results, .{ .wait = true }))); value_buffer.deinit(); try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); try std.testing.expectEqual([2]i32{ 10, 20 }, try weight_buffer.getValue([2]i32, io)); diff --git a/zml/io.zig b/zml/io.zig index 43bfdf23..f80cb443 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -545,7 +545,7 @@ pub const Loader = struct { var results = try exe.results(arena); args.set(.{buffers}); - exe.call(io, args, &results, .{ .wait = true }); + exe.call(io, args, &results, .{ .wait = true }) catch return error.LoadFailed; buffer.* = results.get(Buffer); try self.delivered.put(self.allocator, tensor.id, {}); diff --git a/zml/testing.zig b/zml/testing.zig index 853f7d2e..858d3e2a 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -461,7 +461,7 @@ pub fn testLayer( var results = try allocator.alloc(zml.Buffer, output_count); defer allocator.free(results); - exe.call(io, exe_args, &exe_results, .{ .wait = true }); + try exe.call(io, exe_args, &exe_results, .{ .wait = true }); exe_results.fill(.{results}); defer for (results) |*result| result.deinit(); From 76cc7dcc9845970117329e5662a080ff12820f38 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:58:53 +0100 Subject: [PATCH 015/103] examples/llm: free buffers with mem.deinitBufferized lfm2, llama, qwen3_5 and qwen3_5_moe each freed their weights with a tree of hand-written unloadBuffers functions, one per layer struct, and each LoadedModel carried its own copy of the same loadBuffers. A walker that missed a field leaked its device memory. models.LoadedModel.loadBuffers now bufferizes and loads any model arm with one body, and main, lfm2's session and the lfm2 and llama layer tests free buffers with mem.deinitBufferized, which follows the Bufferized type. The per-model loaders and every unloadBuffers walker are deleted, so no example calls nn.Linear.unloadBuffers anymore. Co-Authored-By: Claude Opus 5.5 --- examples/llm/main.zig | 2 +- examples/llm/models.zig | 41 ++++++--- examples/llm/models/lfm2/model.zig | 106 ---------------------- examples/llm/models/lfm2/session.zig | 4 +- examples/llm/models/lfm2_tests.zig | 8 +- examples/llm/models/llama/model.zig | 72 --------------- examples/llm/models/llama_tests.zig | 8 +- examples/llm/models/qwen3_5/model.zig | 83 ----------------- examples/llm/models/qwen3_5_moe/model.zig | 89 ------------------ 9 files changed, 43 insertions(+), 370 deletions(-) diff --git a/examples/llm/main.zig b/examples/llm/main.zig index b24c022a..23aab061 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -135,7 +135,7 @@ pub fn main(init: std.process.Init) !void { // Load buffers after the model compilation to be sure to give enough room to the autotune. var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); - defer model.unloadBuffers(&model_buffers, allocator); + defer zml.mem.deinitBufferized(allocator, &model_buffers); progress.end(); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 20aa60c7..a738ff76 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -55,27 +55,42 @@ pub const LoadedModel = union(ModelType) { } } + /// Loads the model's weights from the store. Free them with `zml.mem.deinitBufferized`. pub fn loadBuffers( - self: *LoadedModel, + self: *const LoadedModel, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, ) !Buffers { - return switch (self.*) { - inline else => |*m, t| @unionInit( - Buffers, - @tagName(t), - try m.loadBuffers(allocator, io, platform, store, progress), - ), - }; - } + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - switch (self.*) { - inline else => |*loaded, t| loaded.unloadBuffers(&@field(buffers, @tagName(t)), allocator), - } + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + const buffers: Buffers = switch (self.*) { + inline else => |*m, t| b: { + const Model = @TypeOf(m.inner); + var buffers = try zml.mem.bufferize(allocator, Model, &m.inner); + errdefer zml.mem.deinitBufferized(allocator, &buffers); + try loader.load(io, Model, &m.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + break :b @unionInit(Buffers, @tagName(t), buffers); + }, + }; + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; } pub fn compile( diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index a769d599..bee197fa 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -6,8 +6,6 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.lfm); - pub const Config = struct { architectures: []const []const u8, block_auto_adjust_ff_dim: bool, @@ -72,43 +70,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - Model.unloadBuffers(buffers, allocator); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -164,15 +125,6 @@ pub const Model = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - LmHead.unloadBuffers(&self.lm_head); - for (self.layers) |*layer| { - DecoderLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - pub fn forward( self: Model, tokens: zml.Tensor, @@ -292,10 +244,6 @@ pub const LmHead = struct { const tokens = new_tokens.convert(.u32); return .{ .tokens = if (input.tokens.dim(.seq) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng }; } - - pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { - RmsNorm.unloadBuffers(&self.embedding_norm); - } }; pub const OperatorKind = enum { conv, full_attention }; @@ -394,16 +342,6 @@ pub const DecoderLayer = struct { .kv_cache_index = kv_cache_index.reuseBuffer(input.kv_cache_index), }; } - - pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { - switch (self.operator) { - .conv => |*operator| ShortConv.unloadBuffers(operator), - .self_attn => |*operator| Attention.unloadBuffers(operator), - } - RmsNorm.unloadBuffers(&self.operator_norm); - RmsNorm.unloadBuffers(&self.ffn_norm); - Mlp.unloadBuffers(&self.feed_forward); - } }; pub const ConvParameters = struct { is_prefill: bool }; @@ -469,12 +407,6 @@ pub const ShortConv = struct { const output = self.out_proj.forward(y); return .{ output.reuseBuffer(input), cache.reuseBuffer(cache_) }; } - - pub fn unloadBuffers(self: *zml.Bufferized(ShortConv)) void { - Linear.unloadBuffers(&self.in_proj); - Linear.unloadBuffers(&self.out_proj); - self.kernel.deinit(); - } }; pub const Attention = struct { @@ -562,15 +494,6 @@ pub const Attention = struct { return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; } - - pub fn unloadBuffers(self: *zml.Bufferized(Attention)) void { - Linear.unloadBuffers(&self.q_proj); - Linear.unloadBuffers(&self.k_proj); - Linear.unloadBuffers(&self.v_proj); - Linear.unloadBuffers(&self.out_proj); - RmsNorm.unloadBuffers(&self.q_layernorm); - RmsNorm.unloadBuffers(&self.k_layernorm); - } }; pub const Cache = struct { @@ -581,11 +504,6 @@ pub const Cache = struct { return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; } - pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { - ConvCache.unloadBuffers(&self.conv); - KvCache.unloadBuffers(&self.kv); - } - pub fn reuseBuffer(self: Cache, other: Cache) Cache { return .{ .conv = self.conv.reuseBuffer(other.conv), .kv = self.kv.reuseBuffer(other.kv) }; } @@ -606,10 +524,6 @@ pub const ConvCache = struct { return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; } - pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { - self.state.deinit(); - } - pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { return .{ .state = self.state.reuseBuffer(other.state) }; } @@ -628,11 +542,6 @@ pub const KvCache = struct { return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; } - pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { - self.k.deinit(); - self.v.deinit(); - } - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { return .{ .k = self.k.reuseBuffer(other.k), .v = self.v.reuseBuffer(other.v) }; } @@ -668,11 +577,6 @@ pub const Linear = struct { return .{ .weight = weight, .bias = bias, .in_tag = zml.Shape.toTag(tag), .out_tag = out_tag }; } - pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { - linear.weight.deinit(); - if (linear.bias) |*bias| bias.deinit(); - } - pub fn forward(self: Linear, x: zml.Tensor) zml.Tensor { var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; @@ -692,12 +596,6 @@ const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - Linear.unloadBuffers(&self.w1); - Linear.unloadBuffers(&self.w2); - Linear.unloadBuffers(&self.w3); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); } @@ -712,10 +610,6 @@ const RmsNorm = struct { return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index ecc5abf8..df617bec 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -31,7 +31,7 @@ pub const Session = struct { ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); - errdefer model.Cache.unloadBuffers(&cache_buffers); + errdefer zml.mem.deinitBufferized(allocator, &cache_buffers); var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); @@ -62,7 +62,7 @@ pub const Session = struct { pub fn deinit(self: *Session) void { self.prefill.deinit(self.allocator); self.decode.deinit(self.allocator); - model.Cache.unloadBuffers(&self.cache_buffers); + zml.mem.deinitBufferized(self.allocator, &self.cache_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buf); self.generated_token_slice.free(self.allocator); } diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index b9424a04..0d6d9354 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -49,8 +49,12 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); - defer repo_model.unloadBuffers(&model_buffers, allocator); + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, .parallelism = 16 }); + defer loader.deinit(); + var model_buffers = try zml.mem.bufferize(allocator, lfm2.Model, &repo_model.inner); + defer zml.mem.deinitBufferized(allocator, &model_buffers); + try loader.load(io, lfm2.Model, &repo_model.inner, &model_buffers, &store, .{ .progress = &progress }); + try loader.await(io); const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { .stablehlo => .{ .stablehlo = {} }, diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index ee84ea79..0dfcbb18 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -6,8 +6,6 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.llama); - pub const Config = struct { bos_token_id: u32, eos_token_id: EosTokens, @@ -69,43 +67,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - if (buffers.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); - Llama.unloadBuffers(&buffers.model, allocator); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -161,11 +122,6 @@ pub const Model = struct { self.model.deinit(allocator); } - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - if (self.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); - Llama.unloadBuffers(&self.model, allocator); - } - fn lmHead(self: Model) LmHead { return .init(self); } @@ -238,15 +194,6 @@ const Llama = struct { allocator.free(self.layers); } - pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - RmsNorm.unloadBuffers(&self.norm); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. pub fn forward( @@ -364,13 +311,6 @@ pub const TransformerLayer = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - SelfAttn.unloadBuffers(&self.self_attn); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - Mlp.unloadBuffers(&self.mlp); - } - pub const Input = struct { layer: TransformerLayer, hidden: zml.Tensor, @@ -429,10 +369,6 @@ const RmsNorm = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); @@ -454,10 +390,6 @@ const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const proj = self.up_proj.forward(x, x.dtype()); var output = self.gate_proj.forward(x, x.dtype()); @@ -499,10 +431,6 @@ const SelfAttn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - /// Self zml.attention. /// - If token_index is set, x is assumed to be the representation of one new token, /// and kv_cache will be read for the previous tokens. diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 92828c53..aea9c965 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -47,8 +47,12 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); - defer repo_model.unloadBuffers(&model_buffers, allocator); + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, .parallelism = 16 }); + defer loader.deinit(); + var model_buffers = try zml.mem.bufferize(allocator, llama.Model, &repo_model.inner); + defer zml.mem.deinitBufferized(allocator, &model_buffers); + try loader.load(io, llama.Model, &repo_model.inner, &model_buffers, &store, .{ .progress = &progress }); + try loader.await(io); progress.end(); try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 30b29f6a..d3901378 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -6,8 +6,6 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.qwen3_5); - pub const Config = struct { text_config: TextConfig, }; @@ -74,44 +72,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - TextModel.unloadBuffers(&buffers.text_model, allocator); - zml.nn.Linear.unloadBuffers(&buffers.lm_head); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -172,10 +132,6 @@ pub const Model = struct { self.text_model.deinit(allocator); } - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - TextModel.unloadBuffers(&self.text_model, allocator); - zml.nn.Linear.unloadBuffers(&self.lm_head); - } pub fn forward( self: Model, tokens_: zml.Tensor, @@ -289,15 +245,6 @@ pub const TextModel = struct { allocator.free(self.layers); } - pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); - } - pub fn forward( self: TextModel, tokens: zml.Tensor, @@ -368,16 +315,6 @@ pub const TransformerLayer = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - switch (self.attn) { - .full_attention => |*self_attn| SelfAttn.unloadBuffers(self_attn), - .linear_attention => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), - } - Mlp.unloadBuffers(&self.mlp); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - } - pub fn forward( self: TransformerLayer, x0: zml.Tensor, @@ -489,10 +426,6 @@ pub const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const up_projed = self.up_proj.forward(x, x.dtype()); const gate = self.gate_proj.forward(x, x.dtype()); @@ -544,10 +477,6 @@ pub const SelfAttn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const q_proj = self.q_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); const q, var gate = q_proj.chunkExact(.hd, 2); @@ -765,10 +694,6 @@ pub const GatedDeltaNet = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - zml.Buffer.deinitAll(GatedDeltaNet, self); - } - fn recurrentGatedDeltaRule( query: zml.Tensor, key: zml.Tensor, @@ -944,10 +869,6 @@ pub const RmsNorm = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const weight_f32 = self.weight.convert(.f32); @@ -965,10 +886,6 @@ pub const RmsNormGated = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 84e6db0b..122f4965 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -100,43 +100,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - TextModel.unloadBuffers(&buffers.text_model, allocator); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -188,9 +151,6 @@ pub const Model = struct { self.text_model.deinit(allocator); } - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - TextModel.unloadBuffers(&self.text_model, allocator); - } pub fn forward( self: Model, tokens_: zml.Tensor, @@ -298,16 +258,6 @@ pub const TextModel = struct { allocator.free(self.layers); } - pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer, allocator); - } - allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); - zml.nn.Linear.unloadBuffers(&self.lm_head); - } - pub fn sampler(self: TextModel) Sampler { return .{ .norm = self.norm, @@ -404,16 +354,6 @@ pub const TransformerLayer = struct { self.moe.deinit(allocator); } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer), allocator: std.mem.Allocator) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - switch (self.attn) { - .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), - .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), - } - Moe.unloadBuffers(&self.moe, allocator); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - } - pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { const self = input.layer; const x0 = input.hidden; @@ -549,10 +489,6 @@ pub const SelfAttn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const q_proj = self.q_proj.forward(x, x.dtype()) .splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); @@ -655,10 +591,6 @@ pub const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const up_projed = self.up_proj.forward(x, x.dtype()); const gate = self.gate_proj.forward(x, x.dtype()); @@ -684,10 +616,6 @@ const Router = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Router)) void { - zml.Buffer.deinitAll(Router, self); - } - pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const router_logits = self.router.forward(x, .f32); const routing = router_logits.topK(.{ .topk = .expert }, self.num_experts_per_tok, .{}); @@ -764,11 +692,6 @@ pub const Moe = struct { return moe_output.add(shared); } - - pub fn unloadBuffers(self: *zml.Bufferized(Moe), allocator: std.mem.Allocator) void { - _ = allocator; - zml.Buffer.deinitAll(Moe, self); - } }; pub const TextRotaryEmbedding = struct { @@ -894,10 +817,6 @@ pub const GatedDeltaNet = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - zml.Buffer.deinitAll(GatedDeltaNet, self); - } - fn recurrentGatedDeltaRule( query: zml.Tensor, key: zml.Tensor, @@ -1063,10 +982,6 @@ pub const RmsNorm = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const weight_f32 = self.weight.convert(.f32); @@ -1084,10 +999,6 @@ pub const RmsNormGated = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); From 0fc28bf0bcfad13aa469d3a6344d88f560f983cc Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:03:15 +0100 Subject: [PATCH 016/103] zml/mem: take bufferize's model without a type bufferize(allocator, T, &model) repeated the model's type next to a pointer that already carries it. It becomes bufferize(allocator, &model) and returns Bufferized of the pointed-to type. The callers in examples/io, examples/llm and both mnist examples move with it. Buffer.deinitAll keeps its type argument. llmd's bufferize calls follow in its next zml bump. Co-Authored-By: Claude Opus 5.5 --- examples/io/main.zig | 2 +- examples/llm/models.zig | 2 +- examples/llm/models/lfm2_tests.zig | 2 +- examples/llm/models/llama_tests.zig | 2 +- examples/mnist/mnist.zig | 2 +- examples/mnist/mnist_data_parallel.zig | 2 +- zml/mem.zig | 17 +++++++++-------- 7 files changed, 15 insertions(+), 14 deletions(-) diff --git a/examples/io/main.zig b/examples/io/main.zig index 396cfdfd..8e1ef82e 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -203,7 +203,7 @@ pub fn main(init: std.process.Init) !void { const now: std.Io.Timestamp = .now(io, .awake); - var buffers = try zml.mem.bufferize(allocator, AllTensorsModel, &model); + var buffers = try zml.mem.bufferize(allocator, &model); defer { for (buffers.tensors) |*b| b.deinit(); allocator.free(buffers.tensors); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index a738ff76..851dd856 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -77,7 +77,7 @@ pub const LoadedModel = union(ModelType) { const buffers: Buffers = switch (self.*) { inline else => |*m, t| b: { const Model = @TypeOf(m.inner); - var buffers = try zml.mem.bufferize(allocator, Model, &m.inner); + var buffers = try zml.mem.bufferize(allocator, &m.inner); errdefer zml.mem.deinitBufferized(allocator, &buffers); try loader.load(io, Model, &m.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 0d6d9354..4791baad 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -51,7 +51,7 @@ pub fn main(init: std.process.Init) !void { var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, .parallelism = 16 }); defer loader.deinit(); - var model_buffers = try zml.mem.bufferize(allocator, lfm2.Model, &repo_model.inner); + var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); defer zml.mem.deinitBufferized(allocator, &model_buffers); try loader.load(io, lfm2.Model, &repo_model.inner, &model_buffers, &store, .{ .progress = &progress }); try loader.await(io); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index aea9c965..f0027a74 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -49,7 +49,7 @@ pub fn main(init: std.process.Init) !void { var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, .parallelism = 16 }); defer loader.deinit(); - var model_buffers = try zml.mem.bufferize(allocator, llama.Model, &repo_model.inner); + var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); defer zml.mem.deinitBufferized(allocator, &model_buffers); try loader.load(io, llama.Model, &repo_model.inner, &model_buffers, &store, .{ .progress = &progress }); try loader.await(io); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index a6937405..c28fcea7 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -44,7 +44,7 @@ const Mnist = struct { platform: *const zml.Platform, store: *const zml.io.TensorStore, ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, Mnist, self); + var buffers = try zml.mem.bufferize(allocator, self); errdefer unloadBuffers(&buffers); var loader: zml.io.Loader = try .init(allocator, platform, .default); diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index 54acc13e..d955809c 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -38,7 +38,7 @@ const Mnist = struct { platform: *const zml.Platform, store: *const zml.io.TensorStore, ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, Mnist, self); + var buffers = try zml.mem.bufferize(allocator, self); errdefer zml.Buffer.deinitAll(Mnist, &buffers); var loader: zml.io.Loader = try .init(allocator, platform, .default); diff --git a/zml/mem.zig b/zml/mem.zig index 44aca8e0..888e8064 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -371,10 +371,11 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu } } -/// Convert a model to its bufferized form by replacing Tensor fields with Buffer -/// and allocating any required slices using the provided allocator. -pub inline fn bufferize(allocator: std.mem.Allocator, comptime ModelType: type, model: *const ModelType) !Bufferized(ModelType) { - var bufferized: Bufferized(ModelType) = undefined; +/// Convert the model `model` points to into its bufferized form by replacing +/// Tensor fields with Buffer and allocating any required slices using the +/// provided allocator. +pub inline fn bufferize(allocator: std.mem.Allocator, model: anytype) !Bufferized(@TypeOf(model.*)) { + var bufferized: Bufferized(@TypeOf(model.*)) = undefined; try bufferizeInner(allocator, model.*, &bufferized); return bufferized; } @@ -420,7 +421,7 @@ test "bufferize maps a tuple's fields as Bufferized renumbers them" { const b: Tensor = .init(.{3}, .f16); var tensors = [_]Tensor{ a, b }; const model: Model = .{ 3, a, &tensors, .{ b, a } }; - var bufferized = try bufferize(std.testing.allocator, Model, &model); + var bufferized = try bufferize(std.testing.allocator, &model); defer deinitBufferized(std.testing.allocator, &bufferized); try std.testing.expect(bufferized[0].shape().eql(a.shape())); try std.testing.expectEqual(2, bufferized[1].len); @@ -438,7 +439,7 @@ test "bufferize rolls back earlier slice fields on allocation failure" { const model: Model = .{ .first = &tensors, .second = &tensors }; var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 1 }); - if (bufferize(failing.allocator(), Model, &model)) |result| { + if (bufferize(failing.allocator(), &model)) |result| { var unexpected = result; deinitBufferized(failing.allocator(), &unexpected); return error.ExpectedOutOfMemory; @@ -461,7 +462,7 @@ test "deinitBufferized frees recursive const slices" { .{ .weights = weights[1..2] }, }; const model: Model = .{ .layers = &layers, .fixed = undefined }; - var bufferized = try bufferize(std.testing.allocator, Model, &model); + var bufferized = try bufferize(std.testing.allocator, &model); deinitBufferized(std.testing.allocator, &bufferized); } @@ -480,6 +481,6 @@ test "deinitBufferized frees slices under optionals and unions" { .absent = null, .ffns = .{ .{ .dense = .{ .weights = weights[2..3] } }, .{ .experts = &experts }, .none }, }; - var bufferized = try bufferize(std.testing.allocator, Model, &model); + var bufferized = try bufferize(std.testing.allocator, &model); deinitBufferized(std.testing.allocator, &bufferized); } From e7efef66be27420ecb228a6d5f04ff6f4e3f43ca Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:06:48 +0100 Subject: [PATCH 017/103] zml/quantization: add Quantization.Target for block-FP8 layout Block-FP8 weights must be bound in the FP8 encoding and scale layout the platform's GEMMs read, and activations quantized to the same encoding. quantizeInput chose the encoding with its own platform switch, and weights had nothing to follow. Quantization.Target{fp8, swizzled_scales} and Target.of(platform) are now the one switch: FN on CUDA with e8m0 scales swizzled on sm_10x, FNUZ on CDNA3, FN on CDNA4 and RDNA4, and no FP8 elsewhere or when a ROCm device's architecture is unknown. quantizeInput reads Target.of(platform).fp8, so its result is unchanged; on CUDA it now queries the compute capability on each call. The block128 scaled dot test takes its dtype from Target as well. Co-Authored-By: Claude Opus 5.5 --- zml/nn.zig | 2 +- zml/quantization.zig | 41 +++++++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 56fd8b51..00501d55 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -227,7 +227,7 @@ test "block128 scaled dot layouts" { const platform = zml.testing.env(); // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) if (platform.target != .cuda) return error.SkipZigTest; - const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; + const dtype = Quantization.Target.of(platform).fp8.?; const Local = struct { const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; diff --git a/zml/quantization.zig b/zml/quantization.zig index 09688314..66c8ff79 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -105,6 +105,36 @@ pub const Quantization = struct { return self == .mxfp8 or self == .mxfp4; } }; + + /// What a platform's FP8 GEMMs read: the encoding and the scale layout + /// that block-FP8 weights are bound in, and that `quantizeInput` + /// quantizes activations to. + pub const Target = struct { + /// The E4M3 encoding of FP8 weights and activations: FNUZ on CDNA3, + /// FN on CUDA, CDNA4 and RDNA4. Null on other platforms: block-FP8 + /// activations stay BF16 (`quantizeInput`), and weights keep the + /// checkpoint's encoding. + fp8: ?DataType = .f8e4m3fn, + /// Whether e8m0 block scales are laid out in the 128x4 blocks of + /// Blackwell's block-scaled MMA (CUDA sm_10x). + swizzled_scales: bool = false, + + /// `platform`'s target: on CUDA, FN, its scales swizzled on sm_10x; + /// on ROCm, by architecture, null when its compute capability is + /// unknown; elsewhere, CPU included, null and unswizzled. The only + /// platform switch for block-FP8 weights. + pub fn of(platform: *const Platform) Target { + return switch (platform.target) { + .cuda => .{ .swizzled_scales = if (platform_mod.cuda.computeCapability(platform)) |cc| cc.major == 10 else false }, + .rocm => .{ .fp8 = if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, + } else null }, + else => .{ .fp8 = null }, + }; + } + }; }; pub const QuantizedInput = struct { @@ -127,16 +157,7 @@ pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, return switch (quantization.scheme) { .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input.convert(.bf16), global_scale, axis) else null, .fp8_block32, .fp8_block128 => blk: { - const dtype: DataType = switch (platform.target) { - .cuda => .f8e4m3fn, - .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => .f8e4m3fn, - .cdna3 => .f8e4m3fnuz, - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => return null, - } else return null, - else => return null, - }; - + const dtype = Quantization.Target.of(platform).fp8 orelse return null; break :blk switch (quantization.scheme) { .fp8_block32 => quantizeBlockFp8(input, axis, 32, dtype, .f8e8m0), .fp8_block128 => quantizeBlockFp8(input.convert(.bf16), axis, 128, dtype, .f32), From 99d7112bb8675481f01297477d698148476714c2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:08:06 +0100 Subject: [PATCH 018/103] zml/sharding: make Partitioning.Logical a runtime value A partitioning written as a struct of tags, such as .{ .out = .model }, could only be parsed at comptime: Partitioning.parse captured it in a comptime Logical{src, tgt}. Code that picks a partitioning at runtime, such as a weight binder that chooses row or column parallelism, had no value to pass. Logical is now a runtime value, {entries, default}, keyed by tensor axis tags. Each entry holds a Spec: a logical mesh axis, .replicated or .open. Logical.init takes a struct of specs (null optional specs are skipped), a Logical or .replicated; get(tag) returns an axis's spec, and resolve(tags, mesh, default) turns it into a positional Partitioning for one tensor on one mesh, so one value applies to tensors whose axes are ordered differently and to different meshes. Shape.withPartitioning, Tensor.withPartitioning and the old TensorStore's maybeCreateTensor resolve through it, and maybeCreateHostPinnedTensor no longer needs a comptime partitioning. Partitioning.parse, Logical.parse, apply and applyUseAll are deleted. Shape.eqlWithTagsAndPartitioning also compares the mesh and each axis's partition spec. Co-Authored-By: Claude Opus 5.5 --- zml/Sharding.zig | 186 +++++++++++++++++++++++++++-------------------- zml/io.zig | 4 +- zml/shape.zig | 34 ++++++++- zml/tensor.zig | 6 +- 4 files changed, 148 insertions(+), 82 deletions(-) diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 2637f209..bc5e6afe 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -405,90 +405,122 @@ pub const Partitioning = packed struct { try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); } - /// Parse a struct literal into a partitioning object. - /// The struct passed to partitioning must only have fields names present in `tags`. - /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. - /// The default value is important. - /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. - pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { - const logical = if (@TypeOf(partitioning) == Logical) - partitioning - else - // capture all the comptime logic in the Logical struct. - comptime Logical.parse(partitioning); - - return logical.applyUseAll(tags, mesh, default_value); - } - + /// A reusable, mesh-independent partitioning keyed by tensor axis tags. + /// Values name logical mesh axes (eg `.model`), or are `.replicated` / `.open`. + /// It is resolved into a positional `Partitioning` once the tensor tags and the mesh are known, + /// so the same value can be applied to tensors with reordered axes, or on different meshes. pub const Logical = struct { - src: Shape.TagsArray, - tgt: Shape.TagsArray, + pub const Spec = union(enum) { + /// Sharded along the logical mesh axis with this name. + axis: Shape.Tag, + replicated, + open, - const replicated: Shape.Tag = Shape.toTag(.replicated); - const open: Shape.Tag = Shape.toTag(.open); - - pub fn parse(v: anytype) Logical { - const V = @TypeOf(v); - - var src_: Shape.TagsArray = .empty; - var tgt_: Shape.TagsArray = .empty; - - if (comptime stdx.meta.isStruct(V)) { - const field_names = std.meta.fieldNames(V); - stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); - inline for (field_names) |field_name| { - const fv = @field(v, field_name); - tgt_.appendAssumeCapacity(@tagName(fv)); - - if (!comptime stdx.meta.isTuple(V)) { - src_.appendAssumeCapacity(field_name); - } - } - return .{ .src = src_, .tgt = tgt_ }; - } - - stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); - } - - test "Logical.parse" { - const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); - - try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.src.constSlice()); - try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.tgt.constSlice()); - } - - pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { - var partition_: Partitioning = .repeat(default_value, tags.len); - var missing_src_axis: ?u8 = null; - - for (0.., logical.src.slice(), logical.tgt.slice()) |i, shape_tag, mesh_tag| { - const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { - missing_src_axis = @intCast(i); - continue; + pub fn init(value: anytype) Spec { + const V = @TypeOf(value); + if (V == Spec) return value; + if (V == @EnumLiteral()) return switch (value) { + .replicated => .replicated, + .open => .open, + else => .{ .axis = Shape.toTag(value) }, }; - - const spec: PartitionSpec = if (mesh_tag == Logical.replicated) - .replicated - else if (mesh_tag == Logical.open) - .open - else spec: { - const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { - stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); - }; - break :spec .sharded(@intCast(mesh_ax)); - }; - partition_ = partition_.set(shape_ax, spec); + if (V == Shape.Tag) return .{ .axis = value }; + stdx.debug.compileError("Partitioning.Logical.Spec expects a logical axis tag, .replicated or .open, got {}", .{V}); } - return .{ partition_, missing_src_axis }; + + pub fn resolve(self: Spec, mesh: *const Mesh) PartitionSpec { + return switch (self) { + .replicated => .replicated, + .open => .open, + .axis => |t| .sharded(@intCast(mesh.resolveLogicalAxis(t) orelse { + stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, t }); + })), + }; + } + }; + + const Entry = struct { tag: Shape.Tag, spec: Spec }; + + entries: stdx.BoundedArray(Entry, MAX_RANK) = .empty, + /// Spec of the axes without an entry. `null` defers to the default of the caller. + default: ?Spec = null, + + pub const replicated: Logical = .{ .default = .replicated }; + + /// Accepts a `Logical`, `.replicated`, or a struct mapping tensor axis tags to specs, + /// eg `.{ .out = .model, .in = .replicated }`. Null optional specs are skipped. + pub fn init(specs: anytype) Logical { + const T = @TypeOf(specs); + if (T == Logical) return specs; + if (T == @EnumLiteral()) return switch (specs) { + .replicated => Logical.replicated, + else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), + }; + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Partitioning.Logical expects a struct of partition specs or .replicated, got {}", .{T}); + + var result: Logical = .{}; + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const value = @field(specs, field_name); + const spec: ?Spec = if (@typeInfo(@TypeOf(value)) == .optional) (if (value) |v| Spec.init(v) else null) else Spec.init(value); + if (spec) |s| result.entries.append(.{ .tag = Shape.toTag(field_name), .spec = s }) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + } + return result; } - pub fn applyUseAll(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { - const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); - - if (missing_shape_axis) |ax| { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.src.get(ax) }); + /// Returns the spec of the given tensor axis, or `default` when it has no entry. + pub fn get(self: Logical, tag_: anytype) ?Spec { + const axis_tag = Shape.toTag(tag_); + for (self.entries.constSlice()) |entry| { + if (std.mem.eql(u8, std.mem.span(entry.tag), std.mem.span(axis_tag))) return entry.spec; } - return p; + return self.default; + } + + /// Resolves the partitioning of a tensor with the given axis tags on `mesh`. + /// Every entry must name one of `tags`. + pub fn resolve(self: Logical, tags: []const Shape.Tag, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { + var res: Partitioning = .repeat(if (self.default) |d| d.resolve(mesh) else default_value, tags.len); + for (self.entries.constSlice()) |entry| { + const ax = Shape.axisFromTagMaybe(tags, entry.tag) orelse { + stdx.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), entry.tag }); + }; + res = res.set(ax, entry.spec.resolve(mesh)); + } + return res; + } + + test "Logical resolves by tensor axis name on any mesh" { + const tp: Mesh = .initTest(.tp, .{.model}); + const ep: Mesh = .initTest(.ep, .{ .experts, .model }); + const specs: []const Spec = &.{ .init(.model), .replicated, .open }; + for (specs) |spec| { + const logical: Logical = .init(.{ .out = spec, .in = .replicated }); + const out = logical.get(.out).?; + try std.testing.expectEqual(std.meta.activeTag(spec), std.meta.activeTag(out)); + try std.testing.expectEqual(spec.resolve(&ep), out.resolve(&ep)); + try std.testing.expectEqual(null, logical.get(.batch)); + + const tags: []const Shape.Tag = &.{ "in", "batch", "out" }; + try std.testing.expectEqualSlices( + PartitionSpec, + &.{ .replicated, .open, spec.resolve(&tp) }, + logical.resolve(tags, &tp, .open).toArray()[0..3], + ); + try std.testing.expectEqual(spec.resolve(&ep), logical.resolve(tags, &ep, .replicated).get(2)); + } + try std.testing.expectEqual(PartitionSpec.sharded(1), Spec.init(.model).resolve(&ep)); + } + + test "Logical replicated default and optional specs" { + const tp: Mesh = .initTest(.tp, .{.model}); + const logical: Logical = .init(.replicated); + try std.testing.expect(logical.get(.out).? == .replicated); + try std.testing.expectEqual(Partitioning.replicated(2), logical.resolve(&.{ "a", "b" }, &tp, .open)); + + const maybe: ?Spec = null; + const partial: Logical = .init(.{ .a = maybe, .b = @as(?Spec, .init(.model)) }); + try std.testing.expectEqual(1, partial.entries.len); + try std.testing.expectEqual(Partitioning.init(&.{ .open, .mesh_axis_0 }), partial.resolve(&.{ "a", "b" }, &tp, .open)); } }; }; diff --git a/zml/io.zig b/zml/io.zig index f80cb443..2e66af23 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -193,7 +193,7 @@ pub const TensorStore = struct { const resolved_mesh = self.store.mesh(mesh_); // Unspecified tensor axes are treated as replicated const partition: Sharding.Partitioning = if (has_tags) - .parse(parsed_tags.constSlice(), resolved_mesh, .replicated, partitioning) + Sharding.Partitioning.Logical.init(partitioning).resolve(parsed_tags.constSlice(), resolved_mesh, .replicated) else partitioning; return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .{ .mesh = resolved_mesh, .partition = partition }); @@ -204,7 +204,7 @@ pub const TensorStore = struct { stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { const tensor = self.maybeCreateTensor(subkey, tags, mesh_, partitioning); if (tensor) |t| { const storage = self.store.id_to_sources.getPtr(t.id); diff --git a/zml/shape.zig b/zml/shape.zig index b0863367..7f08992e 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -407,6 +407,36 @@ pub const Shape = struct { return self.eql(other) and std.mem.eql(Tag, self.tags(), other.tags()) and self.dtype() == other.dtype(); } + /// Compares dimensions, dtype, tags, mesh, and per-axis partition specs. + pub fn eqlWithTagsAndPartitioning(self: Shape, other: Shape) bool { + if (!self.eqlWithTags(other) or self._sharding.mesh != other._sharding.mesh) return false; + for (0..self.rank()) |ax| { + if (self.partition(ax) != other.partition(ax)) return false; + } + return true; + } + + test eqlWithTagsAndPartitioning { + const other_mesh: Sharding.Mesh = .initTest(.other, .{ .batch, .feature }); + const shape = Shape.init(.{ .out = 8, .in = 16 }, .f32) + .withPartitioning(&test_mesh, .{ .out = .feature, .in = .replicated }); + const Spec = Sharding.Partitioning.Logical.Spec; + // Only the first spec is `shape`'s. + const specs: []const Spec = &.{ .init(.feature), .init(.batch), .replicated, .open }; + for (specs, 0..) |spec, i| { + const other = shape.withPartitioning(&test_mesh, .{ .out = spec, .in = .replicated }); + try testing.expect(shape.eqlWithTags(other)); + try testing.expectEqual(i == 0, shape.eqlWithTagsAndPartitioning(other)); + } + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.withPartitioning(&other_mesh, .{ .out = .feature, .in = .replicated }))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.withDtype(.bf16))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.setDim(.out, 9))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.rename(.{ .out = .features }))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(Shape.init(.{ .out = 8 }, .f32))); + try testing.expect(!shape.withDefaultPartitioning().eqlWithTagsAndPartitioning(Shape.init(.{ .out = 8, .in = 16 }, .f32))); + try testing.expect(Shape.init(.{}, .f32).eqlWithTagsAndPartitioning(Shape.init(.{}, .f32))); + } + /// Format the shape. /// "Shape({.a=10, .b=20}, dtype=.f32)" pub fn format(self: Shape, writer: *std.Io.Writer) !void { @@ -827,9 +857,11 @@ pub const Shape = struct { .grid = undefined, }; + /// Partitions this shape on `mesh` as `Tensor.withPartitioning` takes `partitioning`, except + /// that axes it leaves unspecified are `.replicated`. pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { if (@TypeOf(partitioning) != Sharding.Partitioning) { - return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); + return self.withPartitioning(mesh, Sharding.Partitioning.Logical.init(partitioning).resolve(self.tags(), mesh, .replicated)); } var res = self; res._sharding = .{ .mesh = mesh, .partition = partitioning }; diff --git a/zml/tensor.zig b/zml/tensor.zig index b04c5188..5dd62ddc 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -169,7 +169,9 @@ pub const Tensor = struct { /// Specify the mesh of the input tensor. /// * mesh: *const zml.Mesh, but during compilation the `.meshe_name` syntax can used to get /// a known mesh from the compilation options. - /// * partition spec: a struct where the field names match the axis of the given mesh + /// * partitioning: a `Partitioning`, or anything `Partitioning.Logical.init` accepts: a struct + /// mapping this tensor's axis tags to logical mesh axes, `.replicated` or `.open`. + /// Axes it leaves unspecified are `.open`. /// /// eg `x.withPartitioning(tp, .{ .h = .model }))` or `x.withPartitioning(.tp, .{ .h = .model }))` pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { @@ -179,7 +181,7 @@ pub const Tensor = struct { } if (@TypeOf(partitioning) != Sharding.Partitioning) { - const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, .open, partitioning); + const parsed = Sharding.Partitioning.Logical.init(partitioning).resolve(self._shape.tags(), mesh_, .open); return self.withPartitioning(mesh_, parsed); } From 9503ffedb77560a4c7f9099b9753e226ff6fe6cb Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:13:44 +0100 Subject: [PATCH 019/103] zml/testing: add TensorBytes, writeSafetensorsFile and pattern Tests that read a checkpoint need a safetensors file with known tensors. writeSafetensorsFile writes one from (name, dtype, dims, bytes) entries, laying the tensors out in order after the header, and pattern fills a tensor with distinct, nonzero bytes. Co-Authored-By: Claude Opus 5.5 --- zml/testing.zig | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/zml/testing.zig b/zml/testing.zig index 858d3e2a..9efe82e7 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -365,6 +365,55 @@ pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpec return error.TestExpectedEqual; } +pub const TensorBytes = struct { + name: []const u8, + /// The safetensors dtype name, such as "U16". + dtype: []const u8, + dims: []const i64, + bytes: []const u8, +}; + +/// Writes `tensors` as the safetensors file `sub_path` of `dir`, in order. +pub fn writeSafetensorsFile( + allocator: std.mem.Allocator, + io: std.Io, + dir: std.Io.Dir, + sub_path: []const u8, + tensors: []const TensorBytes, +) !void { + var header: std.Io.Writer.Allocating = .init(allocator); + defer header.deinit(); + const writer = &header.writer; + try writer.writeByte('{'); + var offset: usize = 0; + for (tensors, 0..) |tensor, index| { + if (index != 0) try writer.writeByte(','); + try writer.print("\"{s}\":{{\"dtype\":\"{s}\",\"shape\":[", .{ tensor.name, tensor.dtype }); + for (tensor.dims, 0..) |dim, axis| try writer.print("{s}{d}", .{ if (axis != 0) "," else "", dim }); + try writer.print("],\"data_offsets\":[{d},{d}]}}", .{ offset, offset + tensor.bytes.len }); + offset += tensor.bytes.len; + } + try writer.writeByte('}'); + + const file = try dir.createFile(io, sub_path, .{}); + defer file.close(io); + var header_len: [8]u8 = undefined; + std.mem.writeInt(u64, &header_len, header.written().len, .little); + try file.writePositionalAll(io, &header_len, 0); + var position: u64 = header_len.len; + try file.writePositionalAll(io, header.written(), position); + position += header.written().len; + for (tensors) |tensor| { + try file.writePositionalAll(io, tensor.bytes, position); + position += tensor.bytes.len; + } +} + +/// Distinct, nonzero bytes for test tensor number `seed`. +pub fn pattern(bytes: []u8, seed: u8) void { + for (bytes, 0..) |*byte, index| byte.* = (@as(u8, @truncate(index *% 29)) +% seed *% 53) | 1; +} + fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { var count: usize = 0; var it = store.store.registry.iterator(); From 47786bb0e18b23926343722d9c3b096ee2570ef3 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:13:59 +0100 Subject: [PATCH 020/103] zml/safetensors: delete metadata parsing The registry parsed every header's __metadata__ and every index's metadata into a map of typed values, about 300 lines, and nothing in zml or llmd reads it. Metadata, Metadatas, TensorRegistry.metadata, initWithMetadata, mergeMetadata and the parsing helpers are deleted, along with the registry's unused mutex. A header's __metadata__ is skipped, and the torch2zml howto's sample output loses the debug line that metadata parsing logged. Co-Authored-By: Claude Opus 5.5 --- docs/howtos/howto_torch2zml.md | 1 - zml/safetensors.zig | 304 +-------------------------------- 2 files changed, 3 insertions(+), 302 deletions(-) diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 31584751..32f2ea11 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -160,7 +160,6 @@ bazel build //my_project:torch2zml ./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Llama-3.2-1B-Instruct.activations.safetensors info: Found 564 activations in /Users/corendos/dev/zml/python/temp/Llama-3.2-1B-Instruct.activations.safetensors -debug(zml/safetensors): Added new metadata key=format with value=pt info: Found 146 activations in /Users/corendos/models/Llama-3.2-1B-Instruct/model.safetensors ``` diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 63791ea9..4f3496ea 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -4,10 +4,7 @@ const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; const Dims = Shape.DimsArray; -const StringBuilder = std.ArrayListUnmanaged(u8); - pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); -pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); const log = std.log.scoped(.@"zml/safetensors"); @@ -41,7 +38,7 @@ pub fn fetchRegistry( ); defer safetensors_index.deinit(); - var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); + var registry: TensorRegistry = .init(allocator); try parseSafetensorsIndexFiles( allocator, @@ -253,33 +250,14 @@ pub const TensorRegistry = struct { arena: std.heap.ArenaAllocator, tensors: Tensors, - metadata: Metadatas, - - mutex: std.Io.Mutex = .init, pub fn init(allocator: std.mem.Allocator) TensorRegistry { return .{ .arena = std.heap.ArenaAllocator.init(allocator), .tensors = .{}, - .metadata = .{}, }; } - pub fn initWithMetadata( - allocator: std.mem.Allocator, - metadata: Metadatas, - ) !TensorRegistry { - var self: TensorRegistry = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = .empty, - }; - - try self.mergeMetadata(metadata); - - return self; - } - pub fn fromRepo( allocator: std.mem.Allocator, io: std.Io, @@ -310,45 +288,15 @@ pub const TensorRegistry = struct { pub fn deinit(self: *TensorRegistry) void { const allocator = self.arena.allocator(); self.tensors.deinit(allocator); - self.metadata.deinit(allocator); self.arena.deinit(); } - pub fn mergeMetadata( - self: *TensorRegistry, - other: Metadatas, - ) !void { - // self.mutex.lock(); - // defer self.mutex.unlock(); - - const allocator = self.arena.allocator(); - - var it = other.iterator(); - while (it.next()) |entry| { - const key = try allocator.dupe(u8, entry.key_ptr.*); - const value = try entry.value_ptr.*.clone(allocator); - - const gop = try self.metadata.getOrPut(allocator, key); - if (gop.found_existing) { - gop.value_ptr.*.deinit(allocator); - gop.value_ptr.* = value; - log.debug("Overwrote existing metadata key={s} with value={f}", .{ key, value }); - } else { - gop.value_ptr.* = value; - log.debug("Added new metadata key={s} with value={f}", .{ key, value }); - } - } - } - pub fn registerTensor( self: *TensorRegistry, tensor: Tensor, ) !void { const allocator = self.arena.allocator(); - // self.mutex.lock(); - // defer self.mutex.unlock(); - var tensor_copy = tensor; tensor_copy.name = try allocator.dupe(u8, tensor.name); @@ -423,17 +371,8 @@ pub fn parseSafetensors( const key = entry.key_ptr.*; const value = entry.value_ptr.*; - if (std.mem.eql(u8, key, "__metadata__")) { - // Parse metadata into a local arena to avoid concurrent allocations - // against registry.arena (which is not thread-safe). - var local_arena = std.heap.ArenaAllocator.init(allocator); - defer local_arena.deinit(); - - const metas = try parseMetadata(local_arena.allocator(), value); - try registry.mergeMetadata(metas); - - continue; - } + // Free-form strings, which nothing reads. + if (std.mem.eql(u8, key, "__metadata__")) continue; const shape_field = value.object.get("shape").?.array; @@ -468,13 +407,11 @@ const SafetensorsIndex = struct { arena: std.heap.ArenaAllocator, map: Map, - metadata: Metadatas, pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { return .{ .arena = std.heap.ArenaAllocator.init(allocator), .map = .empty, - .metadata = .empty, }; } @@ -485,7 +422,6 @@ const SafetensorsIndex = struct { pub fn deinit(self: *SafetensorsIndex) void { const allocator = self.arena.allocator(); self.map.deinit(allocator); - self.metadata.deinit(allocator); self.arena.deinit(); } }; @@ -528,10 +464,6 @@ fn parseSafetensorsIndex( log.warn("No weight_map attribute found in index", .{}); } - if (index.object.get("metadata")) |metadata_val| { - safetensors_index.metadata = try parseMetadata(arena_allocator, metadata_val); - } - return safetensors_index; } @@ -612,143 +544,6 @@ pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionE return ModelPathResolutionError.FileNotFound; } -const Metadata = union(enum) { - null: void, - int: i64, - float: f64, - bool: bool, - string: []const u8, - - array_bool: []const bool, - array_int: []const i64, - array_float: []const f64, - array_string: []const []const u8, - - pub const ItemType = enum { - int, - float, - bool, - string, - - pub fn toZigType(comptime kind: ItemType) type { - return switch (kind) { - .int => i64, - .float => f64, - .bool => bool, - .string => []const u8, - }; - } - }; - - pub fn wrap(x: anytype) Metadata { - return switch (@TypeOf(x)) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, - inline f16, f32, f64 => .{ .float = @floatCast(x) }, - bool => .{ .bool = x }, - []const u8 => .{ .string = x }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), - }; - } - - pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { - return switch (@TypeOf(any_slice[0])) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => { - const res = try allocator.alloc(i64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @intCast(val); - return .{ .array_int = res }; - }, - inline f16, f32, f64 => { - const res = try allocator.alloc(f64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @floatCast(val); - return .{ .array_float = res }; - }, - bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, - []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), - }; - } - - pub fn format( - self: Metadata, - writer: *std.Io.Writer, - ) !void { - switch (self) { - .null => _ = try writer.write("null"), - .string => |s| try writer.print("{s}", .{s}), - .bool => |b| try writer.print("{}", .{b}), - .int => |i| try writer.print("{d}", .{i}), - .float => |f| try writer.print("{d}", .{f}), - .array_bool => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{}", .{v}); - } - try writer.writeByte(']'); - }, - .array_int => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_float => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_string => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("\"{s}\"", .{v}); - } - try writer.writeByte(']'); - }, - } - } - - pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { - switch (self) { - .string => |s| allocator.free(s), - .array_bool => |s| allocator.free(s), - .array_int => |s| allocator.free(s), - .array_float => |s| allocator.free(s), - .array_string => |s| { - for (s) |str| allocator.free(str); - allocator.free(s); - }, - else => {}, - } - } - - pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { - return switch (self) { - .null => .null, - .int => |v| .{ .int = v }, - .float => |v| .{ .float = v }, - .bool => |v| .{ .bool = v }, - .string => |s| .{ .string = try allocator.dupe(u8, s) }, - .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, - .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, - .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, - .array_string => |s| blk: { - const new_slice = try allocator.alloc([]const u8, s.len); - errdefer allocator.free(new_slice); - for (s, 0..) |str, i| { - new_slice[i] = try allocator.dupe(u8, str); - } - break :blk .{ .array_string = new_slice }; - }, - }; - } -}; - fn stringToDtype(safetensor_type: []const u8) !DataType { const map = std.StaticStringMap(DataType).initComptime(.{ .{ "F64", .f64 }, @@ -774,96 +569,3 @@ fn stringToDtype(safetensor_type: []const u8) !DataType { return error.UnsupportedDataType; }; } - -fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { - var metadatas: Metadatas = .{}; - var prefix_buf: [1024]u8 = undefined; - var prefix = StringBuilder.initBuffer(&prefix_buf); - - try populateMetadata(allocator, &prefix, val, &metadatas); - - return metadatas; -} - -fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { - const key = prefix.items; - return switch (val) { - .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), - .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), - .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), - .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), - .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), - .array => |v| { - if (v.items.len == 0) return; - if (validSlice(v)) |item_type| { - const data: Metadata = switch (item_type) { - .bool => blk: { - const values = try allocator.alloc(bool, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.bool; - break :blk .{ .array_bool = values }; - }, - .integer => blk: { - const values = try allocator.alloc(i64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.integer; - break :blk .{ .array_int = values }; - }, - .float => blk: { - const values = try allocator.alloc(f64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.float; - break :blk .{ .array_float = values }; - }, - inline .string, .number_string => |tag| blk: { - const values = try allocator.alloc([]const u8, v.items.len); - for (v.items, 0..) |item, i| { - values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); - } - break :blk .{ .array_string = values }; - }, - .null, .array, .object => unreachable, - }; - try metadatas.put(allocator, try allocator.dupe(u8, key), data); - } else { - for (v.items, 0..) |item, i| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try populateMetadata(allocator, prefix, item, metadatas); - prefix.items.len = old_len; - } - } - }, - .object => |v| { - var obj_iter = v.iterator(); - while (obj_iter.next()) |entry| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); - prefix.items.len = old_len; - } - }, - }; -} - -/// We can only create a Zig slice out of json array, if all values -/// in the array have the same type. -fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { - if (v.items.len == 0) return null; - - const item_type: std.meta.Tag(std.json.Value) = v.items[0]; - switch (item_type) { - .null, .array, .object => return null, - else => {}, - } - - for (v.items[1..]) |item| { - if (item != item_type) - return null; - } - - return item_type; -} From 61eeae118b26e9ba0fc2be5c6ad0ee8b950d0458 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:16:32 +0100 Subject: [PATCH 021/103] zml/safetensors: parse headers and the index as typed, checked data A checkpoint's header is untrusted input, but parseSafetensors read it as raw JSON values and unwrapped fields with .?, so a malformed header panicked, or was undefined behavior in release builds. It also cast negative offsets to u64, allocated a header length read from the file without checking it against the file, and never checked that a tensor's data_offsets span its shape. The header is now parsed as typed entries that ignore unknown fields. Its length must fit the file, and each tensor's data_offsets must span exactly the bytes of its shape within the file's data, the byte count checked for overflow. Anything else, including a file shorter than the 8-byte length field, is InvalidSafetensorsHeader, logged with the file and tensor. stringToDtype returns null on an unknown dtype and its caller logs the file and tensor, and the TooManyDimensions log names the file. The index is parsed as a typed {weight_map}. Parsing it as raw values with alloc_if_needed left strings pointing into the reader's buffer, which the next refill overwrote, so an index longer than 64 KiB could name the wrong files. An index without a weight_map is refused rather than read as an empty checkpoint, and a read error returns its cause. Co-Authored-By: Claude Opus 5.5 --- zml/safetensors.zig | 341 ++++++++++++++++++++++++++------------------ 1 file changed, 201 insertions(+), 140 deletions(-) diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 4f3496ea..c33b8a86 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,7 +2,7 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; -const Dims = Shape.DimsArray; +const testing = @import("testing.zig"); pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); @@ -23,42 +23,44 @@ pub fn fetchRegistry( model: std.Io.File, model_path: []const u8, ) !TensorRegistry { - const file_type = resolveFiletype(model_path); + var registry: TensorRegistry = .init(allocator); + switch (resolveFiletype(model_path)) { + .index => { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const scratch = arena.allocator(); - return switch (file_type) { - .index => blk: { - const file_reader_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(file_reader_buf); + var index_uri_buf: [1024]u8 = undefined; + const index_uri = index_uri_buf[0..try model.realPath(io, &index_uri_buf)]; - var index_reader = model.reader(io, file_reader_buf); + var index_reader = model.reader(io, try scratch.alloc(u8, 64 * 1024)); + var json_reader: std.json.Reader = .init(scratch, &index_reader.interface); + // The default allocation, `.alloc_always` over a `json.Reader`, + // copies every string out of `index_reader`'s buffer, which the + // next refill overwrites. + const Index = struct { weight_map: std.json.ArrayHashMap([]const u8) }; + const index = std.json.parseFromTokenSourceLeaky(Index, scratch, &json_reader, .{ .ignore_unknown_fields = true }) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return index_reader.err orelse err, + else => { + log.warn("Invalid safetensors index {s}: needs a weight_map from tensor names to file names ({t})", .{ index_uri, err }); + return error.InvalidSafetensorsHeader; + }, + }; - var safetensors_index = try parseSafetensorsIndex( - allocator, - &index_reader.interface, - ); - defer safetensors_index.deinit(); - - var registry: TensorRegistry = .init(allocator); - - try parseSafetensorsIndexFiles( - allocator, - io, - ®istry, - safetensors_index, - repo, - ); - - break :blk registry; + // The files the weight map names, each read once. + var files: std.StringArrayHashMapUnmanaged(void) = .empty; + for (index.weight_map.map.values()) |file_name| try files.put(scratch, file_name, {}); + for (files.keys()) |file_name| { + const file = try repo.openFile(io, file_name, .{}); + defer file.close(io); + try parseSafetensors(allocator, io, ®istry, file); + } }, - .safetensors => blk: { - var registry: TensorRegistry = .init(allocator); - - try parseSafetensors(allocator, io, ®istry, model); - - break :blk registry; - }, - else => return error.InvalidPath, - }; + .safetensors => try parseSafetensors(allocator, io, ®istry, model), + .unknown => return error.InvalidPath, + } + return registry; } pub const TensorReader = struct { @@ -335,6 +337,10 @@ pub const TensorRegistry = struct { } }; +/// Adds the tensors of the safetensors `file` to `registry`. A header that +/// is not an object of tensors, each with a `dtype`, a `shape` and +/// `data_offsets` spanning exactly its bytes within the file, is +/// `InvalidSafetensorsHeader`, logged. pub fn parseSafetensors( allocator: std.mem.Allocator, io: std.Io, @@ -344,56 +350,81 @@ pub fn parseSafetensors( var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const header_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(header_buf); - - var reader = file.reader(io, header_buf); + var reader = file.reader(io, try arena.allocator().alloc(u8, 64 * 1024)); var file_uri_buf: [1024]u8 = undefined; const file_uri_len = try file.realPath(io, &file_uri_buf); const file_uri = file_uri_buf[0..file_uri_len]; - const json_header_length: u64 = try reader.interface.takeInt(u64, .little); + // `File.Reader` returns `ReadFailed` with its cause in `err`, which a + // failed seek leaves unset. + const json_header_length: u64 = reader.interface.takeInt(u64, .little) catch |err| switch (err) { + error.EndOfStream => { + log.warn("Invalid safetensors header in {s}: shorter than its {d}-byte header length", .{ file_uri, BYTES_HEADER }); + return error.InvalidSafetensorsHeader; + }, + error.ReadFailed => return reader.err orelse err, + }; + // The header and every tensor's data must lie inside the file. + const file_length = try file.length(io); + if (json_header_length > file_length -| BYTES_HEADER) { + log.warn("Invalid safetensors header in {s}: {d} bytes of header in a file of {d}", .{ file_uri, json_header_length, file_length }); + return error.InvalidSafetensorsHeader; + } const data_start_offset = BYTES_HEADER + json_header_length; const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); - try reader.interface.readSliceAll(json_data); + reader.interface.readSliceAll(json_data) catch |err| return switch (err) { + error.ReadFailed => reader.err orelse err, + error.EndOfStream => err, + }; - const headers: std.json.Value = try std.json.parseFromSliceLeaky( - std.json.Value, - arena.allocator(), - json_data, - .{}, - ); + const headers = std.json.parseFromSliceLeaky(std.json.ArrayHashMap(std.json.Value), arena.allocator(), json_data, .{}) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + else => { + log.warn("Invalid safetensors header in {s}: not a JSON object ({t})", .{ file_uri, err }); + return error.InvalidSafetensorsHeader; + }, + }; + const data_len = file_length - data_start_offset; - var it = headers.object.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; + // `data_offsets` are `[start, end)` after the header. + const Entry = struct { dtype: []const u8, shape: []const i64, data_offsets: [2]u64 }; + var it = headers.map.iterator(); + while (it.next()) |kv| { + const key = kv.key_ptr.*; // Free-form strings, which nothing reads. if (std.mem.eql(u8, key, "__metadata__")) continue; - const shape_field = value.object.get("shape").?.array; + const entry = std.json.parseFromValueLeaky(Entry, arena.allocator(), kv.value_ptr.*, .{ .ignore_unknown_fields = true }) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + else => { + log.warn("Invalid safetensors header in {s}: tensor {s} needs a string dtype, an integer shape and two unsigned data_offsets ({t})", .{ file_uri, key, err }); + return error.InvalidSafetensorsHeader; + }, + }; - if (shape_field.items.len > Shape.MAX_RANK) { - log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); + if (entry.shape.len > Shape.MAX_RANK) { + log.warn("Cannot load {s}: tensor {s} has {d} dims, more than {d}", .{ file_uri, key, entry.shape.len, Shape.MAX_RANK }); return error.TooManyDimensions; } + const dtype = stringToDtype(entry.dtype) orelse { + log.warn("Cannot load {s}: tensor {s} has dtype {s}, which zml does not support", .{ file_uri, key, entry.dtype }); + return error.UnsupportedDataType; + }; - const offset_field = value.object.get("data_offsets").?; - const start: u64 = @intCast(offset_field.array.items[0].integer); - const dtype = try stringToDtype(value.object.get("dtype").?.string); - - var dims: Dims = .empty; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); + const start, const end = entry.data_offsets; + const byte_size = checkedByteSize(entry.shape, dtype); + if (byte_size == null or start > end or end > data_len or end - start != byte_size.?) { + log.warn("Invalid safetensors header in {s}: tensor {s} has data_offsets [{d},{d}], which must span the bytes of its {s} shape {any} within the file's {d} data bytes", .{ file_uri, key, start, end, entry.dtype, entry.shape, data_len }); + return error.InvalidSafetensorsHeader; } const tensor: Tensor = .{ .file_uri = file_uri, .name = key, - .shape = .init(dims.slice(), dtype), + .shape = .init(entry.shape, dtype), .offset = data_start_offset + start, }; @@ -401,87 +432,120 @@ pub fn parseSafetensors( } } -const SafetensorsIndex = struct { - pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); - - arena: std.heap.ArenaAllocator, - - map: Map, - - pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .map = .empty, - }; - } - - pub fn iterator(self: *SafetensorsIndex) Map.Iterator { - return self.map.iterator(); - } - - pub fn deinit(self: *SafetensorsIndex) void { - const allocator = self.arena.allocator(); - self.map.deinit(allocator); - self.arena.deinit(); - } -}; - -fn parseSafetensorsIndex( - allocator: std.mem.Allocator, - reader: *std.Io.Reader, -) !SafetensorsIndex { - var safetensors_index: SafetensorsIndex = .init(allocator); - errdefer safetensors_index.deinit(); - - const arena_allocator = safetensors_index.arena.allocator(); - var json_reader: std.json.Reader = .init(arena_allocator, reader); - - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - arena_allocator, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - const weight_map = index.object.get("weight_map"); - - if (weight_map) |wm| { - var it = wm.object.iterator(); - - while (it.next()) |entry| { - const weight_name = entry.key_ptr.*; - const filename = entry.value_ptr.string; - - const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); - - if (!map_entry.found_existing) { - map_entry.value_ptr.* = .empty; - } - - try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); - } - } else { - log.warn("No weight_map attribute found in index", .{}); - } - - return safetensors_index; +/// The bytes of `dims` of `dtype`, as `Shape.byteSize` counts them, or null +/// on a negative dim or a bit count past u64. +fn checkedByteSize(dims: []const i64, dtype: DataType) ?u64 { + var bits: u64 = dtype.bitSizeOf(); + for (dims) |d| bits = std.math.mul(u64, bits, std.math.cast(u64, d) orelse return null) catch return null; + return std.math.divCeil(u64, bits, 8) catch unreachable; } -fn parseSafetensorsIndexFiles( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - safetensors_index: SafetensorsIndex, - repo: std.Io.Dir, -) !void { - const filenames = safetensors_index.map.keys(); - for (filenames) |filename| { - const file = try repo.openFile(io, filename, .{}); +test "parseSafetensors rejects a malformed header or data_offsets that do not span their tensor within the file" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const max = "18446744073709551615"; + // One U8 tensor over the file's 4 data bytes. `header` overrides the + // whole header, `header_len` the header's length field. + const cases = [_]struct { offsets: []const u8 = "[0,4]", shape: []const u8 = "[4]", header: ?[]const u8 = null, header_len: ?u64 = null, valid: bool = false }{ + .{ .valid = true }, + .{ .offsets = "[4,0]" }, + .{ .offsets = "[0,3]" }, + .{ .offsets = "[0,5]" }, + .{ .offsets = "[-4,4]" }, + .{ .offsets = "[0," ++ max ++ "]" }, + .{ .offsets = "[" ++ max ++ "," ++ max ++ "]" }, + .{ .offsets = "[0,18446744073709551616]" }, + .{ .offsets = "[0]" }, + .{ .shape = "[-4]" }, + .{ .shape = "[4294967296,4294967296]" }, + .{ .header = "{\"x\":{\"dtype\":\"U8\",\"shape\":[4]}}" }, + .{ .header = "{\"x\":{\"dtype\":\"U8\",\"shape\":4,\"data_offsets\":[0,4]}}" }, + .{ .header = "[]" }, + .{ .header = "{\"x\":" }, + .{ .header_len = 1 << 20 }, + .{ .header_len = std.math.maxInt(u64) }, + }; + for (cases, 0..) |case, index| { + var header_buffer: [128]u8 = undefined; + const header = case.header orelse try std.fmt.bufPrint(&header_buffer, "{{\"x\":{{\"dtype\":\"U8\",\"shape\":{s},\"data_offsets\":{s}}}}}", .{ case.shape, case.offsets }); + var name_buffer: [32]u8 = undefined; + const file = try tmp.dir.createFile(io, try std.fmt.bufPrint(&name_buffer, "{d}.safetensors", .{index}), .{ .read = true }); defer file.close(io); - try parseSafetensors(allocator, io, registry, file); + var length: [BYTES_HEADER]u8 = undefined; + std.mem.writeInt(u64, &length, case.header_len orelse header.len, .little); + try file.writePositionalAll(io, &length, 0); + try file.writePositionalAll(io, header, length.len); + try file.writePositionalAll(io, &.{ 1, 2, 3, 4 }, length.len + header.len); + + var registry: TensorRegistry = .init(allocator); + defer registry.deinit(); + const result = parseSafetensors(allocator, io, ®istry, file); + if (case.valid) { + try result; + try std.testing.expectEqual(@as(u64, BYTES_HEADER + header.len), registry.tensors.get("x").?.offset); + } else { + try std.testing.expectError(error.InvalidSafetensorsHeader, result); + } } } +test "fetchRegistry refuses a file shorter than its header length or an index without a weight_map" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const files = [_]struct { []const u8, []const u8 }{ + .{ "short.safetensors", &.{ 1, 2, 3 } }, + .{ "list.safetensors.index.json", "[]" }, + .{ "unmapped.safetensors.index.json", "{\"metadata\":{\"total_size\":4}}" }, + .{ "unnamed.safetensors.index.json", "{\"weight_map\":{\"w\":4}}" }, + }; + for (files) |file| { + const name, const data = file; + try tmp.dir.writeFile(io, .{ .sub_path = name, .data = data }); + const model = try tmp.dir.openFile(io, name, .{}); + defer model.close(io); + try std.testing.expectError(error.InvalidSafetensorsHeader, fetchRegistry(allocator, io, tmp.dir, model, name)); + } +} + +test "fromRepo reads every file an index longer than its read buffer names" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const scratch = arena.allocator(); + + // About 90 KiB of weight map, read through a 64 KiB buffer: file names + // straddle refills. + const shards = 16; + const tensors_per_shard = 64; + var index: std.Io.Writer.Allocating = .init(scratch); + try index.writer.writeAll("{\"metadata\":{\"total_size\":1024},\"weight_map\":{"); + for (0..shards) |shard| { + const file_name = try std.fmt.allocPrint(scratch, "model-{d:0>5}-of-{d:0>5}.safetensors", .{ shard + 1, shards }); + var tensors: [tensors_per_shard]testing.TensorBytes = undefined; + for (&tensors, 0..) |*tensor, expert| { + const name = try std.fmt.allocPrint(scratch, "model.layers.{d}.mlp.experts.{d}.down_proj.weight", .{ shard, expert }); + tensor.* = .{ .name = name, .dtype = "U8", .dims = &.{1}, .bytes = &.{0} }; + try index.writer.print("{s}\"{s}\":\"{s}\"", .{ if (shard + expert == 0) "" else ",", name, file_name }); + } + try testing.writeSafetensorsFile(allocator, io, tmp.dir, file_name, &tensors); + } + try index.writer.writeAll("}}"); + try std.testing.expect(index.written().len > 64 * 1024); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = index.written() }); + + var registry: TensorRegistry = try .fromRepo(allocator, io, tmp.dir); + defer registry.deinit(); + // Each file holds its own tensors: a misread file name would lose some. + try std.testing.expectEqual(@as(usize, shards * tensors_per_shard), registry.tensors.count()); +} + pub fn resolveFiletype(path: []const u8) FileType { if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { return .index; @@ -544,7 +608,7 @@ pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionE return ModelPathResolutionError.FileNotFound; } -fn stringToDtype(safetensor_type: []const u8) !DataType { +fn stringToDtype(safetensor_type: []const u8) ?DataType { const map = std.StaticStringMap(DataType).initComptime(.{ .{ "F64", .f64 }, .{ "F32", .f32 }, @@ -564,8 +628,5 @@ fn stringToDtype(safetensor_type: []const u8) !DataType { .{ "BOOL", .bool }, }); - return map.get(safetensor_type) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); - return error.UnsupportedDataType; - }; + return map.get(safetensor_type); } From a85dde6c3b61d6fdd7f6bcef4d561516e8e2a511 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:18:15 +0100 Subject: [PATCH 022/103] zml/safetensors: refuse a duplicate tensor name registerTensor put each tensor in the registry, so when two files of a checkpoint named the same tensor, the last one read silently won and which bytes were loaded depended on the index's file order. It now refuses a name already registered with DuplicateTensor, logging the tensor and both files. A checkpoint that repeats a name across its shards no longer loads. llmd checks the registry before registering a tensor of its own, so it is unaffected. Co-Authored-By: Claude Opus 5.5 --- zml/safetensors.zig | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/zml/safetensors.zig b/zml/safetensors.zig index c33b8a86..e4c481f9 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -293,6 +293,8 @@ pub const TensorRegistry = struct { self.arena.deinit(); } + /// Adds `tensor`, its name and file URI copied; a name already here is + /// `DuplicateTensor`, logged. pub fn registerTensor( self: *TensorRegistry, tensor: Tensor, @@ -304,7 +306,12 @@ pub const TensorRegistry = struct { tensor_copy.name = try allocator.dupe(u8, tensor.name); tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); - try self.tensors.put(allocator, tensor_copy.name, tensor_copy); + const entry = try self.tensors.getOrPut(allocator, tensor_copy.name); + if (entry.found_existing) { + log.warn("Tensor {s} of {s} is already registered from {s}", .{ tensor.name, tensor.file_uri, entry.value_ptr.file_uri }); + return error.DuplicateTensor; + } + entry.value_ptr.* = tensor_copy; } pub fn reader( @@ -340,7 +347,8 @@ pub const TensorRegistry = struct { /// Adds the tensors of the safetensors `file` to `registry`. A header that /// is not an object of tensors, each with a `dtype`, a `shape` and /// `data_offsets` spanning exactly its bytes within the file, is -/// `InvalidSafetensorsHeader`, logged. +/// `InvalidSafetensorsHeader`, and a name already registered +/// `DuplicateTensor`, both logged. pub fn parseSafetensors( allocator: std.mem.Allocator, io: std.Io, @@ -440,7 +448,7 @@ fn checkedByteSize(dims: []const i64, dtype: DataType) ?u64 { return std.math.divCeil(u64, bits, 8) catch unreachable; } -test "parseSafetensors rejects a malformed header or data_offsets that do not span their tensor within the file" { +test "parseSafetensors rejects a malformed header, data_offsets that do not span their tensor within the file, or a name already registered" { const allocator = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); @@ -485,6 +493,9 @@ test "parseSafetensors rejects a malformed header or data_offsets that do not sp if (case.valid) { try result; try std.testing.expectEqual(@as(u64, BYTES_HEADER + header.len), registry.tensors.get("x").?.offset); + // A name already registered, as one repeated across a checkpoint's + // shards. + try std.testing.expectError(error.DuplicateTensor, parseSafetensors(allocator, io, ®istry, file)); } else { try std.testing.expectError(error.InvalidSafetensorsHeader, result); } From 8c01ba2b9b2f1c1a1ce1d47b904bf31a8f757826 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:28:06 +0100 Subject: [PATCH 023/103] zml/safetensors: resolve checkpoints strictly and add addRepo and Tensor.readRange Opening a checkpoint had several faults. fromPath opened a file path relative to the file's own directory, so a relative path such as models/x/model.safetensors looked for models/x/models/x/model.safetensors. resolveModelRepo accepted a file and opened its directory, and mapped every error but FileNotFound to InvalidPath, including a denied directory and a canceled open. resolveModelEntrypoint ignored why each probe failed, and fetchRegistry leaked the registry when a file failed to parse. - fromPath opens a file by its base name in its own directory. - resolveModelRepo takes directories only; anything else is a logged InvalidPath, and PermissionDenied and Canceled pass through. ModelPathResolutionError no longer lists OutOfMemory. - resolveModelEntrypoint becomes private. It still tries model.safetensors after the index, and reports PermissionDenied when neither opens and one was denied: S3 answers 403 for a missing key to a principal that may not list the bucket. - fetchRegistry becomes the private addCheckpoint and the registry is freed when parsing fails. FileType and resolveFiletype are deleted. TensorRegistry.addRepo(io, repo, name) adds a second checkpoint's tensors under name and a dot, so a speculative drafter can share its model's registry; it refuses a name a tensor is already under before reading anything. parseSafetensors takes that prefix and allocates from the registry's allocator. checkpointFiles(arena, io, path) names the safetensors files fromPath reads for a path, in their directory, which it returns open. It reads no header and logs nothing, for a reader that prepares a load before the registry exists. fromPath and it share how a path locates its checkpoint (locateCheckpoint), the directory's entrypoint (resolveModelEntrypoint, which also names it) and the files an index lists (indexFiles). safetensors.Tensor gains read, which reads a whole tensor from its file, and readRange, which reads a tensor-relative range of a file the caller opened with positional, bounds-checked reads. TensorReader stays for now. testing.writeSafetensors writes a fixture file and registers it. Co-Authored-By: Claude Opus 5.5 --- zml/safetensors.zig | 577 +++++++++++++++++++++++++++++++++++--------- zml/testing.zig | 16 ++ 2 files changed, 474 insertions(+), 119 deletions(-) diff --git a/zml/safetensors.zig b/zml/safetensors.zig index e4c481f9..5a10aa1b 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -10,59 +10,6 @@ const log = std.log.scoped(.@"zml/safetensors"); const BYTES_HEADER = 8; -pub const FileType = enum { - index, - safetensors, - unknown, -}; - -pub fn fetchRegistry( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - model: std.Io.File, - model_path: []const u8, -) !TensorRegistry { - var registry: TensorRegistry = .init(allocator); - switch (resolveFiletype(model_path)) { - .index => { - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - const scratch = arena.allocator(); - - var index_uri_buf: [1024]u8 = undefined; - const index_uri = index_uri_buf[0..try model.realPath(io, &index_uri_buf)]; - - var index_reader = model.reader(io, try scratch.alloc(u8, 64 * 1024)); - var json_reader: std.json.Reader = .init(scratch, &index_reader.interface); - // The default allocation, `.alloc_always` over a `json.Reader`, - // copies every string out of `index_reader`'s buffer, which the - // next refill overwrites. - const Index = struct { weight_map: std.json.ArrayHashMap([]const u8) }; - const index = std.json.parseFromTokenSourceLeaky(Index, scratch, &json_reader, .{ .ignore_unknown_fields = true }) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.ReadFailed => return index_reader.err orelse err, - else => { - log.warn("Invalid safetensors index {s}: needs a weight_map from tensor names to file names ({t})", .{ index_uri, err }); - return error.InvalidSafetensorsHeader; - }, - }; - - // The files the weight map names, each read once. - var files: std.StringArrayHashMapUnmanaged(void) = .empty; - for (index.weight_map.map.values()) |file_name| try files.put(scratch, file_name, {}); - for (files.keys()) |file_name| { - const file = try repo.openFile(io, file_name, .{}); - defer file.close(io); - try parseSafetensors(allocator, io, ®istry, file); - } - }, - .safetensors => try parseSafetensors(allocator, io, ®istry, model), - .unknown => return error.InvalidPath, - } - return registry; -} - pub const TensorReader = struct { tensor: Tensor, file: std.Io.File, @@ -223,6 +170,11 @@ pub const TensorReader = struct { } }; +/// A checkpoint tensor: its name, its shape, and where its bytes are, at +/// `offset` of the file `file_uri` names. The URI is what the io that read +/// the header resolved, a VFS URI under a VFS, so read the tensor through +/// an io that resolves it alike. A registry's entries own nothing; the +/// registry owns their strings. pub const Tensor = struct { file_uri: []const u8, name: []const u8, @@ -237,6 +189,33 @@ pub const Tensor = struct { return TensorReader.init(io, self, buffer, opts); } + /// Reads the whole tensor into `buffer`, which holds exactly its bytes + /// (`OutOfBounds` otherwise). Opens `file_uri` through `io` for the + /// call, failing as that io's open does; a file that ends inside the + /// tensor is `UnexpectedEndOfFile`. + pub fn read(self: Tensor, io: std.Io, buffer: []u8) !void { + if (buffer.len != self.byteSize()) return error.OutOfBounds; + const file = try std.Io.Dir.openFile(.cwd(), io, self.file_uri, .{ .mode = .read_only }); + defer file.close(io); + return self.readRange(io, file, buffer, 0); + } + + pub const RangeError = std.Io.File.ReadPositionalError || error{ OutOfBounds, UnexpectedEndOfFile }; + + /// Reads `buffer.len` bytes of the tensor from `tensor_offset`, relative + /// to its data, out of `file`, which the caller opened at `file_uri`. A + /// range past the tensor is `OutOfBounds`, a file that ends inside it + /// `UnexpectedEndOfFile`. The reads are positional, so concurrent calls + /// may share one file. + pub fn readRange(self: Tensor, io: std.Io, file: std.Io.File, buffer: []u8, tensor_offset: u64) RangeError!void { + const tensor_size = self.byteSize(); + if (tensor_offset > tensor_size or buffer.len > tensor_size - tensor_offset) return error.OutOfBounds; + if (buffer.len == 0) return; + const file_offset = std.math.add(u64, self.offset, tensor_offset) catch return error.OutOfBounds; + _ = std.math.add(u64, file_offset, buffer.len) catch return error.OutOfBounds; + if (try file.readPositionalAll(io, buffer, file_offset) != buffer.len) return error.UnexpectedEndOfFile; + } + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ self.name, @@ -248,11 +227,75 @@ pub const Tensor = struct { } }; +test "Tensor.readRange reads tensor-relative ranges within the tensor and its file" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const file = try tmp.dir.createFile(io, "tensor.bin", .{ .read = true }); + defer file.close(io); + try file.writePositionalAll(io, "prefix0123456789", 0); + + var tensor: Tensor = .{ + .file_uri = "unused", + .name = "tensor", + .shape = .init(.{10}, .u8), + .offset = "prefix".len, + }; + var middle: [4]u8 = undefined; + try tensor.readRange(io, file, &middle, 3); + try std.testing.expectEqualStrings("3456", &middle); + + var empty: [0]u8 = .{}; + try tensor.readRange(io, file, &empty, 10); + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &empty, 11)); + + var too_large: [2]u8 = undefined; + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &too_large, 9)); + + // A file that ends inside the tensor. + tensor.shape = .init(.{12}, .u8); + var tail: [3]u8 = undefined; + try std.testing.expectError(error.UnexpectedEndOfFile, tensor.readRange(io, file, &tail, 9)); + + tensor.offset = std.math.maxInt(u64); + var overflow: [1]u8 = undefined; + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &overflow, 0)); + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &overflow, 1)); +} + +test "Tensor.read reads the whole tensor from its file" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const file = try tmp.dir.createFile(io, "tensor.bin", .{ .read = true }); + try file.writePositionalAll(io, "prefix0123456789", 0); + + var path_buffer: [1024]u8 = undefined; + const path_len = try file.realPath(io, &path_buffer); + file.close(io); + + const tensor: Tensor = .{ + .file_uri = path_buffer[0..path_len], + .name = "tensor", + .shape = .init(.{10}, .u8), + .offset = "prefix".len, + }; + var whole: [10]u8 = undefined; + try tensor.read(io, &whole); + try std.testing.expectEqualStrings("0123456789", &whole); + try std.testing.expectError(error.OutOfBounds, tensor.read(io, whole[0..9])); +} + +/// The tensors of one or more checkpoints by name, read from their +/// safetensors headers; no tensor data is read. It owns every name and URI. pub const TensorRegistry = struct { arena: std.heap.ArenaAllocator, tensors: Tensors, + /// An empty registry. pub fn init(allocator: std.mem.Allocator) TensorRegistry { return .{ .arena = std.heap.ArenaAllocator.init(allocator), @@ -260,33 +303,80 @@ pub const TensorRegistry = struct { }; } + /// The checkpoint in `repo`: its `model.safetensors.index.json` and the + /// files it lists, else `model.safetensors`. `repo` stays the caller's + /// to close. pub fn fromRepo( allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, ) !TensorRegistry { - const entrypoint_file, const entrypoint_path = try resolveModelEntrypoint(io, repo); - defer entrypoint_file.close(io); - return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); + const entrypoint, const kind, _ = try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + + var registry: TensorRegistry = .init(allocator); + errdefer registry.deinit(); + try addCheckpoint(io, ®istry, repo, entrypoint, kind, ""); + return registry; } + /// The checkpoint at `path`, read through `io`: a `.safetensors` file, + /// a `.safetensors.index.json` and the files it lists, or a directory + /// holding `model.safetensors.index.json` or `model.safetensors`. A + /// relative path is from the cwd; under a VFS, `path` may be a remote + /// URI. A missing path or entrypoint is `FileNotFound`, a denied + /// directory or entrypoint `PermissionDenied`; a file opened by name, + /// the one given or one an index lists, fails as `io`'s open does (a + /// denied local file is `AccessDenied`). A malformed header, such as + /// one whose offsets leave the file in a truncated download or one + /// shorter than its 8-byte length field, or an index without a + /// `weight_map` from tensor names to file names, is + /// `InvalidSafetensorsHeader`, and a name two of its files repeat + /// `DuplicateTensor`, both logged. pub fn fromPath( allocator: std.mem.Allocator, io: std.Io, path: []const u8, ) !TensorRegistry { - var repo = try resolveModelRepo(io, path); + const repo_path, const file_kind = locateCheckpoint(path); + var repo = try resolveModelRepo(io, repo_path); defer repo.close(io); + const kind = file_kind orelse return fromRepo(allocator, io, repo); - const entrypoint_file, const entrypoint_path = if (std.mem.endsWith(u8, path, ".safetensors.index.json") or std.mem.endsWith(u8, path, ".safetensors")) - .{ try repo.openFile(io, path, .{ .mode = .read_only }), path } - else - try resolveModelEntrypoint(io, repo); - defer entrypoint_file.close(io); + // `repo` is the file's directory: the file opens by its base name. + const entrypoint = try repo.openFile(io, std.fs.path.basename(path), .{ .mode = .read_only }); + defer entrypoint.close(io); - return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); + var registry: TensorRegistry = .init(allocator); + errdefer registry.deinit(); + try addCheckpoint(io, ®istry, repo, entrypoint, kind, ""); + return registry; } + /// Adds every tensor of the checkpoint in `repo`, read as `fromRepo` + /// reads it, under `name` and a dot: `View.withPrefix(name)` reaches + /// them. A second checkpoint, such as a speculative drafter's, joins + /// the registry under a name of its own, and one store then holds both. + /// Adding under a name that a tensor already here is named under fails + /// with `error.DuplicateTensor`, logged, before anything is read. `repo` + /// stays the caller's to close. + pub fn addRepo(self: *TensorRegistry, io: std.Io, repo: std.Io.Dir, name: []const u8) !void { + const prefix = try std.mem.concat(self.arena.child_allocator, u8, &.{ name, "." }); + defer self.arena.child_allocator.free(prefix); + // A checkpoint added beside another owns its prefix: no tensor already + // here may be named under it. + for (self.tensors.keys()) |key| { + if (std.mem.startsWith(u8, key, prefix)) { + log.warn("Cannot add a checkpoint under {s}: tensor {s} is already named under it", .{ name, key }); + return error.DuplicateTensor; + } + } + const entrypoint, const kind, _ = try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + try addCheckpoint(io, self, repo, entrypoint, kind, prefix); + } + + /// Frees every entry, its name and URI. pub fn deinit(self: *TensorRegistry) void { const allocator = self.arena.allocator(); self.tensors.deinit(allocator); @@ -344,18 +434,118 @@ pub const TensorRegistry = struct { } }; -/// Adds the tensors of the safetensors `file` to `registry`. A header that -/// is not an object of tensors, each with a `dtype`, a `shape` and -/// `data_offsets` spanning exactly its bytes within the file, is +/// What a checkpoint's entrypoint is: an index of the safetensors files +/// holding its tensors, or the one file holding them. +const EntrypointKind = enum { index, safetensors }; + +/// Where the checkpoint at `path` is (`TensorRegistry.fromPath`): its +/// directory, and the kind of the file `path` names there, or null when +/// `path` names the directory itself. +fn locateCheckpoint(path: []const u8) struct { []const u8, ?EntrypointKind } { + const kind: ?EntrypointKind = if (std.mem.endsWith(u8, path, ".safetensors.index.json")) + .index + else if (std.mem.endsWith(u8, path, ".safetensors")) + .safetensors + else + null; + return .{ if (kind == null) path else std.fs.path.dirname(path) orelse ".", kind }; +} + +/// Adds the tensors of the checkpoint whose entrypoint is open in `repo` to +/// `registry` under `prefix`. An index's files are read from `repo`. +fn addCheckpoint( + io: std.Io, + registry: *TensorRegistry, + repo: std.Io.Dir, + entrypoint: std.Io.File, + kind: EntrypointKind, + prefix: []const u8, +) !void { + switch (kind) { + .safetensors => return parseSafetensors(io, registry, entrypoint, prefix), + .index => { + var arena = std.heap.ArenaAllocator.init(registry.arena.child_allocator); + defer arena.deinit(); + const scratch = arena.allocator(); + + var index_uri_buf: [1024]u8 = undefined; + const index_uri = index_uri_buf[0..try entrypoint.realPath(io, &index_uri_buf)]; + + var index_reader = entrypoint.reader(io, try scratch.alloc(u8, 64 * 1024)); + const file_names = indexFiles(scratch, &index_reader.interface) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return index_reader.err orelse err, + else => { + log.warn("Invalid safetensors index {s}: needs a weight_map from tensor names to file names ({t})", .{ index_uri, err }); + return error.InvalidSafetensorsHeader; + }, + }; + for (file_names) |file_name| { + const file = try repo.openFile(io, file_name, .{}); + defer file.close(io); + try parseSafetensors(io, registry, file, prefix); + } + }, + } +} + +/// The files a safetensors index maps tensors to, each once, in the order +/// its `weight_map` first names them, allocated in `arena`. An index +/// without a `weight_map` from tensor names to file names fails with the +/// JSON parser's error, unlogged. +fn indexFiles(arena: std.mem.Allocator, index: *std.Io.Reader) ![]const []const u8 { + var json_reader: std.json.Reader = .init(arena, index); + // The default allocation, `.alloc_always` over a `json.Reader`, copies + // every string out of `index`'s buffer, which the next refill overwrites. + const Index = struct { weight_map: std.json.ArrayHashMap([]const u8) }; + const parsed = try std.json.parseFromTokenSourceLeaky(Index, arena, &json_reader, .{ .ignore_unknown_fields = true }); + var files: std.StringArrayHashMapUnmanaged(void) = .empty; + for (parsed.weight_map.map.values()) |file_name| try files.put(arena, file_name, {}); + return files.keys(); +} + +/// The safetensors files `TensorRegistry.fromPath` reads for `path`, named +/// in their directory, which is returned open for the caller to close: the +/// file a `.safetensors` path names, the files a `.safetensors.index.json` +/// lists, else those the directory's `model.safetensors.index.json` lists, +/// or its `model.safetensors`. The names are allocated in `arena`. No +/// header is read and nothing is logged: a failure here is one `fromPath` +/// reports, with its context. +pub fn checkpointFiles(arena: std.mem.Allocator, io: std.Io, path: []const u8) !struct { std.Io.Dir, []const []const u8 } { + const repo_path, const file_kind = locateCheckpoint(path); + const repo = try std.Io.Dir.openDir(.cwd(), io, repo_path, .{}); + errdefer repo.close(io); + const entrypoint, const kind, const name = if (file_kind) |kind| + .{ try repo.openFile(io, std.fs.path.basename(path), .{ .mode = .read_only }), kind, std.fs.path.basename(path) } + else + try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + const names = switch (kind) { + .safetensors => try arena.dupe([]const u8, &.{name}), + .index => names: { + var index_reader = entrypoint.reader(io, try arena.alloc(u8, 64 * 1024)); + break :names indexFiles(arena, &index_reader.interface) catch |err| switch (err) { + error.ReadFailed => return index_reader.err orelse err, + else => |e| return e, + }; + }, + }; + return .{ repo, names }; +} + +/// Adds the tensors of the safetensors `file` to `registry`, each named +/// `prefix` followed by its key in the header. A header that is not an +/// object of tensors, each with a `dtype`, a `shape` and `data_offsets` +/// spanning exactly its bytes within the file, is /// `InvalidSafetensorsHeader`, and a name already registered /// `DuplicateTensor`, both logged. pub fn parseSafetensors( - allocator: std.mem.Allocator, io: std.Io, registry: *TensorRegistry, file: std.Io.File, + prefix: []const u8, ) !void { - var arena = std.heap.ArenaAllocator.init(allocator); + var arena = std.heap.ArenaAllocator.init(registry.arena.child_allocator); defer arena.deinit(); var reader = file.reader(io, try arena.allocator().alloc(u8, 64 * 1024)); @@ -431,7 +621,7 @@ pub fn parseSafetensors( const tensor: Tensor = .{ .file_uri = file_uri, - .name = key, + .name = if (prefix.len == 0) key else try std.mem.concat(arena.allocator(), u8, &.{ prefix, key }), .shape = .init(entry.shape, dtype), .offset = data_start_offset + start, }; @@ -489,20 +679,82 @@ test "parseSafetensors rejects a malformed header, data_offsets that do not span var registry: TensorRegistry = .init(allocator); defer registry.deinit(); - const result = parseSafetensors(allocator, io, ®istry, file); + const result = parseSafetensors(io, ®istry, file, ""); if (case.valid) { try result; try std.testing.expectEqual(@as(u64, BYTES_HEADER + header.len), registry.tensors.get("x").?.offset); // A name already registered, as one repeated across a checkpoint's // shards. - try std.testing.expectError(error.DuplicateTensor, parseSafetensors(allocator, io, ®istry, file)); + try std.testing.expectError(error.DuplicateTensor, parseSafetensors(io, ®istry, file, "")); } else { try std.testing.expectError(error.InvalidSafetensorsHeader, result); } } } -test "fetchRegistry refuses a file shorter than its header length or an index without a weight_map" { +test "addRepo adds a second checkpoint under a prefix" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // The target is one file, the drafter an index over one shard; both name + // their tensor `w`. + const target = try tmp.dir.createDirPathOpen(io, "target", .{}); + defer target.close(io); + try testing.writeSafetensorsFile(allocator, io, target, "model.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, + }); + const drafter = try tmp.dir.createDirPathOpen(io, "drafter", .{}); + defer drafter.close(io); + try testing.writeSafetensorsFile(allocator, io, drafter, "shard.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 5, 6, 7, 8 } }, + }); + try drafter.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"w\":\"shard.safetensors\"}}" }); + + var registry: TensorRegistry = try .fromRepo(allocator, io, target); + defer registry.deinit(); + try registry.addRepo(io, drafter, "drafter"); + try std.testing.expectError(error.DuplicateTensor, registry.addRepo(io, drafter, "drafter")); + + try std.testing.expectEqual(@as(usize, 2), registry.tensors.count()); + const cases = [_]struct { name: []const u8, bytes: [4]u8 }{ + .{ .name = "w", .bytes = .{ 1, 2, 3, 4 } }, + .{ .name = "drafter.w", .bytes = .{ 5, 6, 7, 8 } }, + }; + for (cases) |case| { + const tensor = registry.tensors.get(case.name).?; + try std.testing.expectEqualStrings(case.name, tensor.name); + var bytes: [4]u8 = undefined; + try tensor.read(io, &bytes); + try std.testing.expectEqualSlices(u8, &case.bytes, &bytes); + } +} + +test "fromPath opens a file, an index or a directory by a path relative to the cwd" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "layer.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, + }); + for ([_][]const u8{ "layer.safetensors.index.json", "model.safetensors.index.json" }) |index| { + try tmp.dir.writeFile(io, .{ .sub_path = index, .data = "{\"weight_map\":{\"w\":\"layer.safetensors\"}}" }); + } + + for ([_][]const u8{ "layer.safetensors", "layer.safetensors.index.json", "." }) |name| { + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); + var registry: TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + var bytes: [4]u8 = undefined; + try registry.tensors.get("w").?.read(io, &bytes); + try std.testing.expectEqualSlices(u8, &.{ 1, 2, 3, 4 }, &bytes); + } +} + +test "fromPath refuses a file shorter than its header length or an index without a weight_map" { const allocator = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); @@ -516,9 +768,10 @@ test "fetchRegistry refuses a file shorter than its header length or an index wi for (files) |file| { const name, const data = file; try tmp.dir.writeFile(io, .{ .sub_path = name, .data = data }); - const model = try tmp.dir.openFile(io, name, .{}); - defer model.close(io); - try std.testing.expectError(error.InvalidSafetensorsHeader, fetchRegistry(allocator, io, tmp.dir, model, name)); + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); + try std.testing.expectError(error.InvalidSafetensorsHeader, TensorRegistry.fromPath(allocator, io, path)); } } @@ -557,66 +810,152 @@ test "fromRepo reads every file an index longer than its read buffer names" { try std.testing.expectEqual(@as(usize, shards * tensors_per_shard), registry.tensors.count()); } -pub fn resolveFiletype(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { - return .safetensors; - } else { - return .unknown; +test "checkpointFiles names the files fromPath reads" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const scratch = arena.allocator(); + + // The index names a twice and leaves out c, which sits beside its files + // as a Mistral-format `consolidated.safetensors` does. + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "a.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{1} }, + .{ .name = "x", .dtype = "U8", .dims = &.{1}, .bytes = &.{2} }, + }); + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "b.safetensors", &.{ + .{ .name = "y", .dtype = "U8", .dims = &.{1}, .bytes = &.{3} }, + }); + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "c.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{4} }, + }); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"w\":\"a.safetensors\",\"y\":\"b.safetensors\",\"x\":\"a.safetensors\"}}" }); + const single = try tmp.dir.createDirPathOpen(io, "single", .{}); + defer single.close(io); + try testing.writeSafetensorsFile(allocator, io, single, "model.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{5} }, + }); + + const cases = [_]struct { []const u8, []const []const u8 }{ + .{ ".", &.{ "a.safetensors", "b.safetensors" } }, + .{ "model.safetensors.index.json", &.{ "a.safetensors", "b.safetensors" } }, + .{ "c.safetensors", &.{"c.safetensors"} }, + .{ "single", &.{"model.safetensors"} }, + }; + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + for (cases) |case| { + const name, const expected = case; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); + const repo, const files = try checkpointFiles(scratch, io, path); + defer repo.close(io); + try std.testing.expectEqual(expected.len, files.len); + for (expected, files) |expected_file, file| { + try std.testing.expectEqualStrings(expected_file, file); + (try repo.openFile(io, file, .{})).close(io); + } + // The registry reads these files and no other. + var registry: TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + var read: std.StringArrayHashMapUnmanaged(void) = .empty; + for (registry.tensors.values()) |tensor| try read.put(scratch, std.fs.path.basename(tensor.file_uri), {}); + try std.testing.expectEqual(read.count(), files.len); + for (files) |file| try std.testing.expect(read.contains(file)); } + const missing = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/missing", .{&tmp.sub_path}); + try std.testing.expectError(error.FileNotFound, checkpointFiles(scratch, io, missing)); } pub const ModelPathResolutionError = error{ FileNotFound, + PermissionDenied, InvalidPath, -} || std.mem.Allocator.Error; + Canceled, +}; +/// Opens the model directory at `path`, relative to the cwd. A path that is +/// not a directory, such as a file's, is `InvalidPath`, logged. pub fn resolveModelRepo( io: std.Io, path: []const u8, ) ModelPathResolutionError!std.Io.Dir { - if (std.mem.endsWith(u8, path, ".safetensors.index.json") or - std.mem.endsWith(u8, path, ".safetensors")) - { - const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - - return dir; - } - - const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; + return std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |err| switch (err) { + error.FileNotFound => error.FileNotFound, + error.AccessDenied, error.PermissionDenied => error.PermissionDenied, + error.Canceled => error.Canceled, + else => { + log.warn("{s} is not a model directory: {t}", .{ path, err }); + return error.InvalidPath; + }, }; - - return dir; } -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, []const u8 } { - { - const index_name = "model.safetensors.index.json"; - if (repo.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { - return .{ index_file, "model.safetensors.index.json" }; - } else |_| {} - } +test "resolveModelRepo refuses a file" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors", .data = "" }); - { - const model_name = "model.safetensors"; - if (repo.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { - return .{ model_file, "model.safetensors" }; - } else |_| {} - } + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/model.safetensors", .{&tmp.sub_path}); + try std.testing.expectError(error.InvalidPath, resolveModelRepo(io, path)); +} - return ModelPathResolutionError.FileNotFound; +/// The entrypoint of the model directory `repo`, open, with its kind and +/// its name there: `model.safetensors.index.json`, else `model.safetensors`. +fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, EntrypointKind, []const u8 } { + // When neither opens, a denied probe is what is reported. On S3 it may + // mean absent: a bucket that may not be listed answers 403 for a + // missing key. + var denied = false; + const entrypoints = [_]struct { []const u8, EntrypointKind }{ + .{ "model.safetensors.index.json", .index }, + .{ "model.safetensors", .safetensors }, + }; + for (entrypoints) |entrypoint| { + const name, const kind = entrypoint; + if (repo.openFile(io, name, .{ .mode = .read_only })) |file| { + return .{ file, kind, name }; + } else |err| switch (err) { + error.Canceled => return ModelPathResolutionError.Canceled, + error.AccessDenied, error.PermissionDenied => denied = true, + else => {}, + } + } + return if (denied) ModelPathResolutionError.PermissionDenied else ModelPathResolutionError.FileNotFound; +} + +test "resolveModelEntrypoint falls back past a denied index and reports a denied probe" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const served = try tmp.dir.createDirPathOpen(io, "served", .{}); + defer served.close(io); + try served.writeFile(io, .{ .sub_path = "model.safetensors", .data = "" }); + + // The testing io, but a missing file is `PermissionDenied`: S3 answers + // 403 for a missing key to a principal without `s3:ListBucket`. + var vtable = io.vtable.*; + vtable.dirOpenFile = struct { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + return std.testing.io.vtable.dirOpenFile(userdata, dir, sub_path, options) catch |err| switch (err) { + error.FileNotFound => error.PermissionDenied, + else => err, + }; + } + }.dirOpenFile; + const unlisted: std.Io = .{ .userdata = io.userdata, .vtable = &vtable }; + + const file, const kind, const name = try resolveModelEntrypoint(unlisted, served); + defer file.close(io); + try std.testing.expectEqual(.safetensors, kind); + try std.testing.expectEqualStrings("model.safetensors", name); + // `tmp.dir` holds neither entrypoint. + try std.testing.expectError(error.PermissionDenied, resolveModelEntrypoint(unlisted, tmp.dir)); + try std.testing.expectError(error.FileNotFound, resolveModelEntrypoint(io, tmp.dir)); } fn stringToDtype(safetensor_type: []const u8) ?DataType { diff --git a/zml/testing.zig b/zml/testing.zig index 9efe82e7..e77fb7ae 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -373,6 +373,22 @@ pub const TensorBytes = struct { bytes: []const u8, }; +/// Writes `tensors` as the safetensors file `sub_path` of `dir`, in order, +/// and registers them from it. +pub fn writeSafetensors( + allocator: std.mem.Allocator, + io: std.Io, + dir: std.Io.Dir, + sub_path: []const u8, + registry: *zml.safetensors.TensorRegistry, + tensors: []const TensorBytes, +) !void { + try writeSafetensorsFile(allocator, io, dir, sub_path, tensors); + const file = try dir.openFile(io, sub_path, .{}); + defer file.close(io); + try zml.safetensors.parseSafetensors(io, registry, file, ""); +} + /// Writes `tensors` as the safetensors file `sub_path` of `dir`, in order. pub fn writeSafetensorsFile( allocator: std.mem.Allocator, From 1501fc4078d52575b1a3e1c25f93ee09927ce176 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:30:01 +0100 Subject: [PATCH 024/103] zml/io: construct TensorStore with init and look tensors up with View.get TensorStore.fromRegistry(allocator, ®istry, meshes) becomes init(allocator, ®istry, Options{ meshes, quantization }). The store still borrows the registry: entries registered after init, such as aliases of a fused tensor, are found, and each bound tensor holds a copy of its entry. Options.quantization is the platform's Quantization.Target, which code binding block-FP8 weights reads through View.quantization; that panics when the store has none, so weights cannot be laid out for a different FP8 encoding than the activations. View.get(subkey) returns the checkpoint tensor, or null, with one hash lookup. It replaces getShape, getShapeOpts and hasKey; hasKey scanned every key for one starting with the subkey, so it also matched a tensor's prefix. View.prefix() returns "" at the root instead of null. The examples, the torch2zml howto and testing.testLayer move to init, and the lfm2 and llama layer tests read tensors with get and Tensor.read. llmd's fromRegistry, hasKey and getShape calls follow in its next zml bump. Co-Authored-By: Claude Opus 5.5 --- docs/howtos/howto_torch2zml.md | 4 +- examples/io/main.zig | 2 +- examples/llm/main.zig | 2 +- examples/llm/models/lfm2_tests.zig | 16 ++- examples/llm/models/llama_tests.zig | 18 ++-- examples/llm/models/qwen3_5/model.zig | 2 +- examples/llm/models/qwen3_5_moe/model.zig | 2 +- examples/mnist/mnist.zig | 2 +- examples/mnist/mnist_data_parallel.zig | 2 +- zml/io.zig | 123 ++++++++++++++-------- zml/quantization.zig | 8 +- zml/testing.zig | 9 +- 12 files changed, 110 insertions(+), 80 deletions(-) diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 32f2ea11..69eb19dc 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -189,7 +189,7 @@ convert it to a `TensorStore`: ```zig pub fn main(init: std.process.Init) !void { ... - var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); + var model_store: zml.io.TensorStore = .init(allocator, &model_registry, .{}); defer model_store.deinit(); } ``` @@ -286,7 +286,7 @@ pub const Wrapper = struct { pub fn main(init: std.process.Init) !void { ... - var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); + var activations_store: zml.io.TensorStore = .init(allocator, &activations_registry, .{}); defer activations_store.deinit(); try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); diff --git a/examples/io/main.zig b/examples/io/main.zig index 8e1ef82e..a89351d5 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -171,7 +171,7 @@ pub fn main(init: std.process.Init) !void { ); const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); defer store.deinit(); const AllTensorsModel = struct { diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 23aab061..6ec8382c 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -107,7 +107,7 @@ pub fn main(init: std.process.Init) !void { // Defines how the model's tensors are sharded across the available devices. const meshes: models.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer store.deinit(); const generation: models.GenerationOptions = .{ diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 4791baad..5ee7aa69 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -41,7 +41,7 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer store.deinit(); var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); @@ -89,7 +89,7 @@ pub fn run( var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var activation_store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer activation_store.deinit(); var ctx = TestContext{ @@ -308,16 +308,12 @@ const TestContext = struct { }; fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { - const shape = store.view().getShape(key) orelse return error.NotFound; + const tensor = store.view().get(key) orelse return error.NotFound; - const host_bytes = try allocator.alloc(u8, shape.byteSize()); + const host_bytes = try allocator.alloc(u8, tensor.byteSize()); defer allocator.free(host_bytes); - var io_buffer: [8 * 1024]u8 = undefined; - var reader = try store.view().getReader(key, io, &io_buffer); - defer reader.deinit(); + try tensor.read(io, host_bytes); - _ = try reader.interface.readSliceAll(host_bytes); - - return zml.Buffer.fromBytes(io, platform, shape, host_bytes); + return zml.Buffer.fromBytes(io, platform, tensor.shape, host_bytes); } diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index f0027a74..c27e6bfd 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -39,7 +39,7 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer store.deinit(); var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); @@ -70,7 +70,7 @@ fn run( var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); + var activation_store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = meshes }); defer activation_store.deinit(); try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); @@ -102,7 +102,7 @@ fn testLayer( ) !void { const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); defer allocator.free(in_key); - const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; + const in_shape = (activation_store.get(in_key) orelse return error.NotFound).shape; var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_shape); @@ -138,16 +138,12 @@ fn testLayer( } fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { - const shape = store.getShape(key) orelse return error.NotFound; + const tensor = store.get(key) orelse return error.NotFound; - const host_bytes = try allocator.alloc(u8, shape.byteSize()); + const host_bytes = try allocator.alloc(u8, tensor.byteSize()); defer allocator.free(host_bytes); - var io_buffer: [8 * 1024]u8 = undefined; - var reader = try store.getReader(key, io, &io_buffer); - defer reader.deinit(); + try tensor.read(io, host_bytes); - _ = try reader.interface.readSliceAll(host_bytes); - - return zml.Buffer.fromBytes(io, platform, shape, host_bytes); + return zml.Buffer.fromBytes(io, platform, tensor.shape, host_bytes); } diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index d3901378..a434c489 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -113,7 +113,7 @@ pub const Model = struct { pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer const lm_head_prefix = b: { - if (store.hasKey("lm_head.weight")) break :b "lm_head"; + if (store.get("lm_head.weight") != null) break :b "lm_head"; break :b "model.language_model.embed_tokens"; }; return .{ diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 122f4965..9fa5300f 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -232,7 +232,7 @@ pub const TextModel = struct { config: Config, gen_options: Model.GenOptions, ) !TextModel { - const lm_head_prefix = if (store.root().hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; + const lm_head_prefix = if (store.root().get("lm_head.weight") != null) "lm_head" else "model.language_model.embed_tokens"; const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); errdefer allocator.free(layers); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c28fcea7..d510e730 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -96,7 +96,7 @@ pub fn main(init: std.process.Init) !void { defer platform.deinit(allocator, io); // Init model - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); defer store.deinit(); const mnist_model: Mnist = .init(store.view()); diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index d955809c..e3871218 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -105,7 +105,7 @@ pub fn main(init: std.process.Init) !void { var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); defer profiler.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{dp_mp} }); defer store.deinit(); const mnist_model: Mnist = .init(store.view()); diff --git a/zml/io.zig b/zml/io.zig index 2e66af23..ae41a5b7 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -14,6 +14,7 @@ const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const tracer = @import("profiling/tracer.zig"); +const Quantization = @import("quantization.zig").Quantization; const safetensors = @import("safetensors.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); @@ -32,15 +33,30 @@ pub const TensorStore = struct { registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh, + quantization: ?Quantization.Target, id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, - pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { + pub const Options = struct { + /// The meshes `View.createTensor` and `mesh` name. The slice is + /// copied; the meshes must outlive the store. + meshes: []const *const Sharding.Mesh = &.{}, + /// How the platform's FP8 GEMMs want block-FP8 weights, for the code + /// that binds them (`View.quantization`): `.of(platform)`. Required to + /// bind block-FP8 weights. + quantization: ?Quantization.Target = null, + }; + + /// A store over `registry`, which it borrows: the registry must outlive + /// the store, and may still gain entries, which views then find. Each + /// tensor the store binds holds a copy of its entry. + pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, opts: Options) TensorStore { var arena: std.heap.ArenaAllocator = .init(allocator); return .{ .registry = registry, - .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), + .meshes = arena.allocator().dupe(*const Sharding.Mesh, opts.meshes) catch @panic("OOM"), + .quantization = opts.quantization, .id_to_sources = .empty, .allocator = allocator, .arena = arena, @@ -97,11 +113,6 @@ pub const TensorStore = struct { return self.id_to_sources.get(id); } - pub fn getShape(self: *const TensorStore, key: []const u8) ?Shape { - const entry_ptr = self.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; - } - pub fn view(self: *TensorStore) View { return .{ .store = self }; } @@ -115,7 +126,7 @@ pub const TensorStore = struct { } std.debug.panic( \\Found no meshes named {s} in TensorStore. - \\Try passing more meshes to `zml.TensorStore.fromRegistry`. + \\Try passing more meshes to `zml.io.TensorStore.init`. \\Known meshes: {f} , .{ name_slice, stdx.fmt.slice(store.meshes) }); } @@ -133,7 +144,8 @@ pub const TensorStore = struct { } pub fn parent(self: *const View) View { - const slice = self.prefix() orelse unreachable; + const slice = self.prefix(); + std.debug.assert(slice.len != 0); const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); var buffer: [256]u8 = undefined; @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); @@ -146,7 +158,7 @@ pub const TensorStore = struct { pub fn withPrefix(self: *const View, prefix_: []const u8) View { var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }); + const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix(), prefix_ }); return .{ .store = self.store, @@ -157,7 +169,7 @@ pub const TensorStore = struct { pub fn withLayer(self: *const View, index: usize) View { var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }); + const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix(), index }); return .{ .store = self.store, @@ -166,22 +178,32 @@ pub const TensorStore = struct { }; } - pub fn prefix(self: *const View) ?[]const u8 { - return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + /// The keys' prefix under this view, with its trailing dot; "" at the + /// root. + pub fn prefix(self: *const View) []const u8 { + return self.prefix_buffer[0..self.prefix_length]; } - pub fn hasKey(self: *const View, subkey: []const u8) bool { + /// The checkpoint tensor `subkey`, the registry's own entry; null + /// when the checkpoint has none. The pointer is valid until the + /// registry next gains an entry. + pub fn get(self: View, subkey: []const u8) ?*const safetensors.Tensor { var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - return for (self.store.registry.tensors.keys()) |k| { - if (std.mem.startsWith(u8, k, key)) break true; - } else false; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); + return self.store.registry.tensors.getPtr(key); + } + + /// See `Options.quantization`. Panics when the store has none: binding + /// block-FP8 weights requires it. + pub fn quantization(self: View) Quantization.Target { + return self.store.quantization orelse + stdx.debug.panic("TensorStore.Options.quantization is unset: block-FP8 weights bind for a platform; pass .of(platform)", .{}); } /// Creates a zml.Tensor from a specific entry in the store. pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); } pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { @@ -201,7 +223,7 @@ pub const TensorStore = struct { pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); } pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { @@ -218,12 +240,12 @@ pub const TensorStore = struct { const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.slice() else null, .replicated) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); } fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); const source = self.store.dupeSource(key) orelse return null; const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); @@ -252,7 +274,7 @@ pub const TensorStore = struct { var buffer: [256]u8 = undefined; for (sources) |subkey| { - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); const tensor = self.store.dupeSource(key) orelse return null; tensor_list.appendAssumeCapacity(tensor); } @@ -266,31 +288,15 @@ pub const TensorStore = struct { return tensor; } - pub fn getShape(self: View, subkey: []const u8) ?Shape { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - return self.store.getShape(key); - } - - pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { - var buffer: [256]u8 = undefined; - const key = if (opts.no_prefix) - subkey - else b: { - break :b makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - }; - return self.store.getShape(key); - } - pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { var key_buffer: [256]u8 = undefined; - const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix(), subkey }); return self.store.getReader(key, io, buffer); } pub fn count(self: View) usize { var count_: usize = 0; - const prefix_ = self.prefix() orelse ""; + const prefix_ = self.prefix(); var it = self.store.registry.tensors.iterator(); while (it.next()) |item| { const key = item.key_ptr.*; @@ -552,6 +558,39 @@ pub const Loader = struct { } }; +test "View.get finds exact keys in the registry the store borrows" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try @import("testing.zig").writeSafetensors(allocator, io, tmp.dir, "layer.safetensors", ®istry, &.{ + .{ .name = "layer.experts.0.down_proj.weight", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, + }); + var store: TensorStore = .init(allocator, ®istry, .{ .quantization = .{ .fp8 = .f8e4m3fnuz } }); + defer store.deinit(); + + const layer = store.view().withPrefix("layer"); + const entry = layer.get("experts.0.down_proj.weight").?; + try std.testing.expectEqual(entry, registry.tensors.getPtr("layer.experts.0.down_proj.weight").?); + var bytes: [4]u8 = undefined; + try entry.read(io, &bytes); + try std.testing.expectEqualSlices(u8, &.{ 1, 2, 3, 4 }, &bytes); + // Exact keys only: a key's prefix names no tensor. + try std.testing.expectEqual(null, layer.get("experts.0.down_proj")); + try std.testing.expectEqualStrings("layer.", layer.prefix()); + try std.testing.expectEqualStrings("", layer.root().prefix()); + try std.testing.expectEqual(1, layer.withPrefix("experts").count()); + try std.testing.expectEqual(0, layer.withPrefix("gate").count()); + try std.testing.expect(layer.quantization().fp8.? == .f8e4m3fnuz); + + // An entry registered after `init`, such as an alias of a fused + // tensor, is found too. + try registry.registerTensor(.{ .file_uri = entry.file_uri, .name = "layer.gate.weight", .shape = entry.shape, .offset = entry.offset }); + try std.testing.expect(layer.get("gate.weight") != null); +} + test "Loader.loadExecute cleans up donated and non-donated inputs" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -567,7 +606,7 @@ test "Loader.loadExecute cleans up donated and non-donated inputs" { defer registry.deinit(); try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); defer store.deinit(); var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); defer loader.deinit(); diff --git a/zml/quantization.zig b/zml/quantization.zig index 66c8ff79..9053b26c 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -106,9 +106,11 @@ pub const Quantization = struct { } }; - /// What a platform's FP8 GEMMs read: the encoding and the scale layout - /// that block-FP8 weights are bound in, and that `quantizeInput` - /// quantizes activations to. + /// What a platform's FP8 GEMMs read, so block-FP8 weights can be bound + /// for it once. Binding them requires the store to carry one + /// (`zml.io.TensorStore.Options.quantization`); the code that binds them + /// reads it (`TensorStore.View.quantization`) and converts or lays them + /// out while binding. pub const Target = struct { /// The E4M3 encoding of FP8 weights and activations: FNUZ on CDNA3, /// FN on CUDA, CDNA4 and RDNA4. Null on other platforms: block-FP8 diff --git a/zml/testing.zig b/zml/testing.zig index e77fb7ae..3f115f51 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -531,17 +531,14 @@ pub fn testLayer( defer for (results) |*result| result.deinit(); var failed: bool = false; - var reader_buffer: [4096]u8 = undefined; for (0..output_count) |i| { const expected_slice = s: { var buffer: [256]u8 = undefined; const subkey = try std.fmt.bufPrint(&buffer, "{d}", .{i}); - var reader = try store_output.getReader(subkey, io, &reader_buffer); - defer reader.deinit(); - const shape = store_output.getShape(subkey).?; - const expected_slice: zml.Slice = try .alloc(allocator, shape); + const tensor = store_output.get(subkey) orelse return error.TensorNotFound; + const expected_slice: zml.Slice = try .alloc(allocator, tensor.shape); errdefer expected_slice.free(allocator); - try reader.interface.readSliceAll(expected_slice.data()); + try tensor.read(io, expected_slice.data()); break :s expected_slice; }; defer expected_slice.free(allocator); From a6c2e28dc31b80683feffa019116c8ca50896b3c Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:40:38 +0100 Subject: [PATCH 025/103] vfs: register backends with read hints A backend is now registered as a Backend: its io together with ReadHints, the smallest request worth sending and whether the backend's latency calls for many requests in flight. Local files read best in 8 MiB requests, the https, s3 and gs backends in 16 MiB and hf in 32 MiB, the remote ones all high-latency. registerBackend replaces register and unregister, and VFSBase is no longer exported. readHints and isLocal answer for a path without opening it, and fromIo finds the VFS behind an Io, so a reader that holds only an Io can size its reads. examples/io and examples/llm register through registerBackend. This breaks llmd's register calls in main.zig and zoo.zig. Co-Authored-By: Claude Opus 5.5 --- examples/io/main.zig | 10 ++-- examples/llm/main.zig | 8 +-- vfs/base.zig | 21 ++++++++ vfs/gcs.zig | 8 +++ vfs/hf.zig | 11 ++++ vfs/http.zig | 8 +++ vfs/s3.zig | 8 +++ vfs/vfs.zig | 123 ++++++++++++++++++++++++++++++------------ 8 files changed, 153 insertions(+), 44 deletions(-) diff --git a/examples/io/main.zig b/examples/io/main.zig index a89351d5..3bd3c3ea 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -47,11 +47,11 @@ pub fn main(init: std.process.Init) !void { var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.register("file", vfs_file.io()); - try vfs.register("https", vfs_https.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); - try vfs.register("gs", gcs_vfs.io()); + try vfs.registerBackend("file", .{ .io = vfs_file.io(), .read_hints = .local }); + try vfs.registerBackend("https", vfs_https.backend()); + try vfs.registerBackend("hf", hf_vfs.backend()); + try vfs.registerBackend("s3", s3_vfs.backend()); + try vfs.registerBackend("gs", gcs_vfs.backend()); const io = vfs.io(); diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 6ec8382c..9126a744 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -71,10 +71,10 @@ pub fn main(init: std.process.Init) !void { var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.register("file", vfs_file.io()); - try vfs.register("gs", gcs_vfs.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); + try vfs.registerBackend("file", .{ .io = vfs_file.io(), .read_hints = .local }); + try vfs.registerBackend("gs", gcs_vfs.backend()); + try vfs.registerBackend("hf", hf_vfs.backend()); + try vfs.registerBackend("s3", s3_vfs.backend()); const io = vfs.io(); diff --git a/vfs/base.zig b/vfs/base.zig index 1e66ca89..6d97b584 100644 --- a/vfs/base.zig +++ b/vfs/base.zig @@ -2,6 +2,27 @@ const std = @import("std"); const stdx = @import("stdx"); +/// How a backend's files are best read (`VFS.readHints`). No one source +/// request size suits every backend: warm local 8 MiB requests on one B70 +/// reached 27.05 GiB/s against 21.33 at 32 MiB, with a quarter of the pinned +/// memory; real AWS plateaued near 950 MiB/s, and 32 MiB was no faster. +pub const ReadHints = struct { + /// Local files, through a VFS or not. + pub const local: ReadHints = .{ .read_chunk_size = 8 * 1024 * 1024 }; + + /// Smallest source request expected to avoid excessive per-request cost + /// or backend rate limiting. + read_chunk_size: usize = 16 * 1024 * 1024, + /// The backend benefits from opening enough source calls to hide request + /// latency before the first response arrives. + high_latency: bool = false, +}; + +pub const Backend = struct { + io: std.Io, + read_hints: ReadHints = .{}, +}; + pub const VFSBase = struct { inner: std.Io, diff --git a/vfs/gcs.zig b/vfs/gcs.zig index 8169c2da..fab055e6 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/gcs"); @@ -531,6 +532,13 @@ pub const GCS = struct { }; } + pub fn backend(self: *GCS) Backend { + return .{ + .io = self.io(), + .read_hints = .{ .high_latency = true }, + }; + } + fn openHandle(self: *GCS) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); diff --git a/vfs/hf.zig b/vfs/hf.zig index 5e6e767a..d1a61a6d 100644 --- a/vfs/hf.zig +++ b/vfs/hf.zig @@ -4,6 +4,7 @@ const stdx = @import("stdx"); const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/hf"); @@ -383,6 +384,16 @@ pub const HF = struct { }; } + pub fn backend(self: *HF) Backend { + return .{ + .io = self.io(), + .read_hints = .{ + .read_chunk_size = 32 * 1024 * 1024, + .high_latency = true, + }, + }; + } + fn openHandle(self: *HF) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); diff --git a/vfs/http.zig b/vfs/http.zig index a98dc5c6..b1bc7b20 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/http"); @@ -94,6 +95,13 @@ pub const HTTP = struct { }; } + pub fn backend(self: *HTTP) Backend { + return .{ + .io = self.io(), + .read_hints = .{ .high_latency = true }, + }; + } + fn openHandle(self: *HTTP) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); diff --git a/vfs/s3.zig b/vfs/s3.zig index 46198ed2..d896c400 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -4,6 +4,7 @@ const stdx = @import("stdx"); const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/s3"); @@ -483,6 +484,13 @@ pub const S3 = struct { }; } + pub fn backend(self: *S3) Backend { + return .{ + .io = self.io(), + .read_hints = .{ .high_latency = true }, + }; + } + fn openHandle(self: *S3) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); diff --git a/vfs/vfs.zig b/vfs/vfs.zig index f32cd447..f2792554 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -1,3 +1,11 @@ +//! A `std.Io` over local files and remote objects. A bare path is served by +//! the inner io; a URI such as `hf://org/model/model.safetensors` or +//! `s3://bucket/key` by the backend registered for its scheme +//! (`registerBackend`). Readers take `io()` and need not know which they +//! read; one that must (to size its reads) asks `fromIo`. +//! +//! The VFS, and every backend registered on it, must outlive each file +//! opened through it. const std = @import("std"); const stdx = @import("stdx"); @@ -7,7 +15,10 @@ pub const GCS = @import("gcs.zig").GCS; pub const HF = @import("hf.zig").HF; pub const HTTP = @import("http.zig").HTTP; pub const S3 = @import("s3.zig").S3; -pub const VFSBase = @import("base.zig").VFSBase; +const base_module = @import("base.zig"); +pub const Backend = base_module.Backend; +pub const ReadHints = base_module.ReadHints; +const VFSBase = base_module.VFSBase; test { _ = @import("gcs.zig"); @@ -33,7 +44,7 @@ const Handle = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, -backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, +backends: std.StringArrayHashMapUnmanaged(Backend) = .empty, handles: stdx.SegmentedList(Handle, 128) = .{}, closed_handles: std.ArrayList(u32) = .empty, @@ -62,49 +73,72 @@ pub fn deinit(self: *VFS) void { self.backends.deinit(self.allocator); } -pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { +/// Serves `scheme` with `backend`, replacing any backend registered under +/// it. `scheme` is kept, not copied: it must outlive the VFS. +pub fn registerBackend(self: *VFS, scheme: []const u8, backend: Backend) std.mem.Allocator.Error!void { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); try self.backends.put(self.allocator, scheme, backend); } -pub fn unregister(self: *VFS, scheme: []const u8) bool { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - return self.backends.remove(scheme); -} - +/// The `std.Io` of this VFS. It points into the VFS, which must not move +/// once this was called. pub fn io(self: *VFS) std.Io { return .{ .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), + .vtable = &io_vtable, }; } +/// The VFS behind an `Io`, or null for any other implementation: a reader +/// that opened a file through `any_io` asks this VFS about that handle. +pub fn fromIo(any_io: std.Io) ?*VFS { + if (any_io.vtable != &io_vtable) return null; + return @fieldParentPtr("base", VFSBase.as(any_io.userdata)); +} + +const io_vtable: std.Io.VTable = VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, +}); + +/// The read hints of the backend serving `path`, for a reader sizing its +/// requests. A path resolves as it does for `openFile`: a bare path is local +/// (`ReadHints.local`), and a scheme no backend serves is +/// `VFSNotRegistered`. +pub fn readHints(self: *VFS, path: []const u8) !ReadHints { + const backend_idx, _, _ = try self.lookupDir(.cwd(), path); + const index = backend_idx orelse return .local; + return self.backends.entries.items(.value)[index].read_hints; +} + +/// Whether `path` resolves to the inner `Io`'s files; an unregistered scheme +/// does not. Opens nothing. +pub fn isLocal(self: *VFS, path: []const u8) bool { + const backend_idx, _, _ = self.lookupDir(.cwd(), path) catch return false; + return backend_idx == null; +} + fn openHandle(self: *VFS) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -145,7 +179,7 @@ fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { } fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { - if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; + if (backend_idx) |idx| return self.backends.entries.items(.value)[idx].io else return self.base.inner; } fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { @@ -443,6 +477,25 @@ fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std. } } +test "VFS gives the read hints of local and registered paths" { + var filesystem = try VFS.init(std.testing.allocator, std.testing.io); + defer filesystem.deinit(); + const hints: ReadHints = .{ .read_chunk_size = 32 * 1024 * 1024, .high_latency = true }; + try filesystem.registerBackend("test", .{ .io = std.testing.io, .read_hints = hints }); + + try std.testing.expectEqual(hints, try filesystem.readHints("test://bucket/object")); + try std.testing.expect(!filesystem.isLocal("test://bucket/object")); + + for ([_][]const u8{ "/tmp/model.safetensors", "models/model.safetensors" }) |path| { + try std.testing.expectEqual(ReadHints{ .read_chunk_size = 8 * 1024 * 1024, .high_latency = false }, try filesystem.readHints(path)); + try std.testing.expect(filesystem.isLocal(path)); + } + try std.testing.expectError(error.VFSNotRegistered, filesystem.readHints("missing://bucket/object")); + try std.testing.expect(!filesystem.isLocal("missing://bucket/object")); + try std.testing.expectEqual(@as(?*VFS, &filesystem), VFS.fromIo(filesystem.io())); + try std.testing.expectEqual(@as(?*VFS, null), VFS.fromIo(std.testing.io)); +} + test "standard streams resolve to their own descriptors" { var vfs: VFS = try .init(std.testing.allocator, std.testing.io); defer vfs.deinit(); @@ -480,7 +533,7 @@ test "copy a local file through the VFS" { defer file_backend.deinit(); var vfs: VFS = try .init(allocator, process_io); defer vfs.deinit(); - try vfs.register("file", file_backend.io()); + try vfs.registerBackend("file", .{ .io = file_backend.io(), .read_hints = .local }); const vfs_io = vfs.io(); { From 8656be1e95e020361dcbc34b4a0f8f09c2daacaa Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:45:15 +0100 Subject: [PATCH 026/103] vfs: serve file:// natively and delete VFS.File Local files are the VFS's own: a file:// URI now resolves to the inner io, like a bare path, and registerBackend asserts that nobody registers the file scheme. vfs/file.zig goes. It was a forwarding backend with a second handle table that, for file:// URIs, also opened a second O_DIRECT descriptor and read through it transparently whenever a read happened to be aligned. Bare paths, which llmd and the examples use, never had that, so only file:// URIs change: they now read buffered. The examples stop creating and registering a VFS.File. This breaks llmd's use of VFS.File in main.zig and zoo.zig. Co-Authored-By: Claude Opus 5.5 --- examples/io/main.zig | 4 - examples/llm/main.zig | 4 - vfs/BUILD.bazel | 1 - vfs/file.zig | 430 ------------------------------------------ vfs/vfs.zig | 58 ++++-- 5 files changed, 45 insertions(+), 452 deletions(-) delete mode 100644 vfs/file.zig diff --git a/examples/io/main.zig b/examples/io/main.zig index 3bd3c3ea..79835286 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -29,9 +29,6 @@ pub fn main(init: std.process.Init) !void { try http_client.initDefaultProxies(allocator, init.environ_map); defer http_client.deinit(); - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); defer vfs_https.deinit(); @@ -47,7 +44,6 @@ pub fn main(init: std.process.Init) !void { var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.registerBackend("file", .{ .io = vfs_file.io(), .read_hints = .local }); try vfs.registerBackend("https", vfs_https.backend()); try vfs.registerBackend("hf", hf_vfs.backend()); try vfs.registerBackend("s3", s3_vfs.backend()); diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 9126a744..83c61c90 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -53,9 +53,6 @@ pub fn main(init: std.process.Init) !void { // // Virtual File Systems // - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; defer http_client.deinit(); @@ -71,7 +68,6 @@ pub fn main(init: std.process.Init) !void { var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.registerBackend("file", .{ .io = vfs_file.io(), .read_hints = .local }); try vfs.registerBackend("gs", gcs_vfs.backend()); try vfs.registerBackend("hf", hf_vfs.backend()); try vfs.registerBackend("s3", s3_vfs.backend()); diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index c8de1e0c..18f62787 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -4,7 +4,6 @@ zig_library( name = "vfs", srcs = [ "base.zig", - "file.zig", "gcs.zig", "hf.zig", "http.zig", diff --git a/vfs/file.zig b/vfs/file.zig deleted file mode 100644 index 1b077c33..00000000 --- a/vfs/file.zig +++ /dev/null @@ -1,430 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const VFSBase = @import("base.zig").VFSBase; - -const log = std.log.scoped(.@"zml/vfs/file"); - -pub const DirectIoError = error{ - UnsupportedPlatform, - MisalignedBuffer, - BufferTooSmall, - UnexpectedFcntlResult, -}; - -fn canUseDirectIO() bool { - if (builtin.target.os.tag == .linux) { - return @hasField(std.posix.O, "DIRECT"); - } - return false; -} - -fn getStatusFlags(file: std.Io.File) DirectIoError!c_int { - while (true) { - const result: c_int = std.posix.system.fcntl(file.handle, std.posix.F.GETFL, @as(c_int, 0)); - switch (std.posix.errno(result)) { - .SUCCESS => return result, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - -fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { - while (true) { - switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { - .SUCCESS => return, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - -fn useDirectIO(file: std.Io.File) DirectIoError!bool { - if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); - const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - return (flags & direct_flag) != 0; - } else { - return DirectIoError.UnsupportedPlatform; - } -} - -fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { - if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); - const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - if ((flags & direct_flag) == 0) return; - - try setStatusFlags(file, flags & ~direct_flag); - } else { - return DirectIoError.UnsupportedPlatform; - } -} - -fn switchToDirectIO(file: std.Io.File) DirectIoError!void { - if (builtin.os.tag == .linux and canUseDirectIO()) { - const flags = try getStatusFlags(file); - const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - try setStatusFlags(file, flags | direct_flag); - } else { - return DirectIoError.UnsupportedPlatform; - } -} - -pub const File = struct { - pub const Config = struct { - direct_io: bool = true, - direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), - }; - - const Handle = struct { - inner_handle: std.Io.File.Handle, - inner_flags: std.Io.File.Flags, - direct_io_handle: ?std.Io.File.Handle, - - pub fn innerFile(self: *Handle) std.Io.File { - return .{ - .handle = self.inner_handle, - .flags = self.inner_flags, - }; - } - - pub fn directIoFile(self: *Handle) std.Io.File { - return .{ - .handle = self.direct_io_handle.?, - .flags = self.inner_flags, - }; - } - }; - - allocator: std.mem.Allocator, - mutex: std.Io.Mutex = .init, - handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .empty, - config: Config, - base: VFSBase, - - pub fn init(allocator: std.mem.Allocator, inner: std.Io, config: Config) File { - return .{ - .allocator = allocator, - .config = config, - .base = .init(inner), - }; - } - - pub fn deinit(self: *File) void { - self.handles.deinit(self.allocator); - self.closed_handles.deinit(self.allocator); - } - - pub fn io(self: *File) std.Io { - return .{ - .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), - }; - } - - fn openHandle(self: *File) !struct { u32, *Handle } { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (self.closed_handles.pop()) |idx| { - return .{ idx, self.handles.at(idx) }; - } - return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; - } - - fn closeHandle(self: *File, idx: u32) !void { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - try self.closed_handles.append(self.allocator, idx); - } - - fn getFileHandle(self: *File, file: std.Io.File) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - return self.handles.at(@intCast(file.handle)); - } - - fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { - const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { - else => return std.Io.File.OpenError.Unexpected, - }; - - return self.config.direct_io and flags.mode == .read_only and file_stat.size >= self.config.direct_io_alignment.toByteUnits(); - } - - fn innerFile(self: *File, file: std.Io.File, data: []const []u8, position: u64) std.Io.File { - const handle = self.getFileHandle(file); - - if (handle.direct_io_handle == null) { - return handle.innerFile(); - } - - const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); - - if (!std.mem.isAligned(@as(usize, position), alignment_bytes)) { - return handle.innerFile(); - } - - var total_size: usize = 0; - for (data) |buf| { - if (!std.mem.isAligned(@intFromPtr(buf.ptr), alignment_bytes)) { - return handle.innerFile(); - } - total_size += buf.len; - } - - if (!std.mem.isAligned(total_size, alignment_bytes)) { - return handle.innerFile(); - } - - return handle.directIoFile(); - } - - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle = self.getFileHandle(o.file); - const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); - const inner_file = self.innerFile(o.file, o.data, position); - return self.base.inner.vtable.operate(self.base.inner.userdata, .{ - .file_read_streaming = .{ - .file = inner_file, - .data = o.data, - }, - }); - }, - .file_write_streaming => |o| { - const handle = self.getFileHandle(o.file); - return self.base.inner.vtable.operate(self.base.inner.userdata, .{ - .file_write_streaming = .{ - .file = handle.innerFile(), - .header = o.header, - .data = o.data, - .splat = o.splat, - }, - }); - }, - .device_io_control, .net_receive, .net_read, .net_send, .net_write => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } - } - - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirOpenDir(self.base.inner.userdata, dir, sub_path, options); - } - - fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirStat(self.base.inner.userdata, dir); - } - - fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirStatFile(self.base.inner.userdata, dir, sub_path, options); - } - - fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); - } - - fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); - errdefer inner_file.close(self.base.inner); - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .inner_handle = inner_file.handle, - .inner_flags = inner_file.flags, - .direct_io_handle = null, - }; - return .{ .handle = @intCast(idx), .flags = inner_file.flags }; - } - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); - errdefer inner_file.close(self.base.inner); - - const use_direct_io = try self.canOpenWithDirectIO(inner_file, flags); - - var direct_io_handle: ?std.Io.File.Handle = null; - if (use_direct_io) { - const direct_io_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); - defer if (direct_io_handle == null) direct_io_file.close(self.base.inner); - - switchToDirectIO(direct_io_file) catch |err| switch (err) { - DirectIoError.UnsupportedPlatform => { - log.debug("Direct I/O is not supported on this platform", .{}); - }, - else => { - log.err("Failed to switch to Direct I/O mode: {any}", .{err}); - return std.Io.File.OpenError.Unexpected; - }, - }; - - if (useDirectIO(direct_io_file) catch false) { - direct_io_handle = direct_io_file.handle; - } - } - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .inner_handle = inner_file.handle, - .inner_flags = inner_file.flags, - .direct_io_handle = direct_io_handle, - }; - - return .{ .handle = @intCast(idx), .flags = inner_file.flags }; - } - - fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirClose(self.base.inner.userdata, dirs); - } - - fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirRead(self.base.inner.userdata, reader, entries); - } - - fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirRealPath(self.base.inner.userdata, dir, out_buffer); - } - - fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirRealPathFile(self.base.inner.userdata, dir, path_name, out_buffer); - } - - fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileStat(self.base.inner.userdata, handle.innerFile()); - } - - fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileLength(self.base.inner.userdata, handle.innerFile()); - } - - fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - for (files) |file| { - const handle = self.getFileHandle(file); - - if (handle.direct_io_handle) |_| { - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.directIoFile()}); - } - - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.innerFile()}); - - self.closeHandle(@intCast(file.handle)) catch unreachable; - } - } - - fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); - } - - fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); - } - - fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); - } - - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = self.innerFile(file, data, offset); - return self.base.inner.vtable.fileReadPositional(self.base.inner.userdata, inner_file, data, offset); - } - - fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, handle.innerFile(), relative_offset); - } - - fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, handle.innerFile(), absolute_offset); - } - - fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, handle.innerFile(), out_buffer); - } -}; diff --git a/vfs/vfs.zig b/vfs/vfs.zig index f2792554..930e0d6a 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -1,8 +1,9 @@ -//! A `std.Io` over local files and remote objects. A bare path is served by -//! the inner io; a URI such as `hf://org/model/model.safetensors` or -//! `s3://bucket/key` by the backend registered for its scheme -//! (`registerBackend`). Readers take `io()` and need not know which they -//! read; one that must (to size its reads) asks `fromIo`. +//! A `std.Io` over local files and remote objects. A bare path or a +//! `file://` URI is served by the inner io; a URI such as +//! `hf://org/model/model.safetensors` or `s3://bucket/key` by the backend +//! registered for its scheme (`registerBackend`). Readers take `io()` and +//! need not know which they read; one that must (to size its reads) asks +//! `fromIo`. //! //! The VFS, and every backend registered on it, must outlive each file //! opened through it. @@ -10,7 +11,6 @@ const std = @import("std"); const stdx = @import("stdx"); -pub const File = @import("file.zig").File; pub const GCS = @import("gcs.zig").GCS; pub const HF = @import("hf.zig").HF; pub const HTTP = @import("http.zig").HTTP; @@ -29,6 +29,9 @@ const log = std.log.scoped(.@"zml/vfs"); /// Indices 0-2 are the standard streams, so `std.Io.File.stdout()` and /// friends resolve to their own descriptor; the cwd placeholder follows. const CWD_HANDLE: u32 = 3; +/// Local files are the VFS's own: a bare path or a `file://` URI resolves +/// to the inner `Io`, never to a registered backend. +const local_scheme = "file"; const VFS = @This(); const Handle = struct { @@ -76,6 +79,7 @@ pub fn deinit(self: *VFS) void { /// Serves `scheme` with `backend`, replacing any backend registered under /// it. `scheme` is kept, not copied: it must outlive the VFS. pub fn registerBackend(self: *VFS, scheme: []const u8, backend: Backend) std.mem.Allocator.Error!void { + std.debug.assert(!std.mem.eql(u8, scheme, local_scheme)); // file is the VFS's own (local_scheme) self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -123,9 +127,9 @@ const io_vtable: std.Io.VTable = VFSBase.vtable(.{ }); /// The read hints of the backend serving `path`, for a reader sizing its -/// requests. A path resolves as it does for `openFile`: a bare path is local -/// (`ReadHints.local`), and a scheme no backend serves is -/// `VFSNotRegistered`. +/// requests. A path resolves as it does for `openFile`: a bare path or a +/// `file://` URI is local (`ReadHints.local`), and a scheme no backend +/// serves is `VFSNotRegistered`. pub fn readHints(self: *VFS, path: []const u8) !ReadHints { const backend_idx, _, _ = try self.lookupDir(.cwd(), path); const index = backend_idx orelse return .local; @@ -190,6 +194,7 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz if (sub_path) |sp| { if (std.mem.indexOf(u8, sp, "://") != null) { const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; + if (std.mem.eql(u8, uri.scheme, local_scheme)) return .{ null, std.Io.Dir.cwd(), self.base.inner }; const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { if (std.mem.eql(u8, uri.scheme, s)) break idx; } else return error.VFSNotRegistered; @@ -486,7 +491,7 @@ test "VFS gives the read hints of local and registered paths" { try std.testing.expectEqual(hints, try filesystem.readHints("test://bucket/object")); try std.testing.expect(!filesystem.isLocal("test://bucket/object")); - for ([_][]const u8{ "/tmp/model.safetensors", "models/model.safetensors" }) |path| { + for ([_][]const u8{ "/tmp/model.safetensors", "models/model.safetensors", "file:///tmp/model.safetensors" }) |path| { try std.testing.expectEqual(ReadHints{ .read_chunk_size = 8 * 1024 * 1024, .high_latency = false }, try filesystem.readHints(path)); try std.testing.expect(filesystem.isLocal(path)); } @@ -496,6 +501,35 @@ test "VFS gives the read hints of local and registered paths" { try std.testing.expectEqual(@as(?*VFS, null), VFS.fromIo(std.testing.io)); } +test "VFS serves bare paths and file:// URIs from the inner Io" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const created = try tmp.dir.createFile(testing_io, "bare.bin", .{ .read = true }); + try created.writePositionalAll(testing_io, "bare path bytes", 0); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; + created.close(testing_io); + + var filesystem = try VFS.init(std.testing.allocator, testing_io); + defer filesystem.deinit(); + const vfs_io = filesystem.io(); + + var uri_buffer: [std.fs.max_path_bytes + 8]u8 = undefined; + const uri = try std.fmt.bufPrint(&uri_buffer, "file://{s}", .{path}); + for ([_][]const u8{ path, uri }) |open_path| { + var real_buffer: [std.fs.max_path_bytes]u8 = undefined; + try std.testing.expectEqualStrings(path, real_buffer[0..try std.Io.Dir.cwd().realPathFile(vfs_io, open_path, &real_buffer)]); + const opened = try std.Io.Dir.openFile(.cwd(), vfs_io, open_path, .{ .mode = .read_only }); + defer opened.close(vfs_io); + const handle, _ = filesystem.getFileHandle(opened); + try std.testing.expectEqual(@as(?usize, null), handle.backend_idx); + var contents: [15]u8 = undefined; + try std.testing.expectEqual(contents.len, try opened.readPositionalAll(vfs_io, &contents, 0)); + try std.testing.expectEqualStrings("bare path bytes", &contents); + } +} + test "standard streams resolve to their own descriptors" { var vfs: VFS = try .init(std.testing.allocator, std.testing.io); defer vfs.deinit(); @@ -529,11 +563,9 @@ test "copy a local file through the VFS" { const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); defer allocator.free(destination_uri); - var file_backend: File = .init(allocator, process_io, .{}); - defer file_backend.deinit(); + // `file://` is the VFS's own scheme: no backend to register. var vfs: VFS = try .init(allocator, process_io); defer vfs.deinit(); - try vfs.registerBackend("file", .{ .io = file_backend.io(), .read_hints = .local }); const vfs_io = vfs.io(); { From 6b8a96d161d8b92a091f11afc222d669d0847df0 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:47:58 +0100 Subject: [PATCH 027/103] vfs: add VFS.Remotes and use it in the examples Every program that reads remote checkpoints wired the same remote schemes by hand: one std.http.Client, then the HTTP, HF, S3 and GCS backends and their registration. The copies had drifted: examples/llm had no https and ignored the proxy variables. VFS.Remotes builds one client on the VFS's base io, with proxies from the environment, sets up https, hf, s3 and gs through each backend's auto constructor and registers all four, or none on error. It is initialized in place, since the backends point at its client. examples/io and examples/llm use it, so examples/llm gains https and proxies. Co-Authored-By: Claude Opus 5.5 --- examples/io/main.zig | 24 ++---------- examples/llm/main.zig | 18 ++------- vfs/vfs.zig | 91 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 94 insertions(+), 39 deletions(-) diff --git a/examples/io/main.zig b/examples/io/main.zig index 79835286..03e708b6 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -24,30 +24,12 @@ pub fn main(init: std.process.Init) !void { const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; const path = it.next() orelse return error.MissingPath; - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - - try http_client.initDefaultProxies(allocator, init.environ_map); - defer http_client.deinit(); - - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); - defer vfs_https.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); - defer gcs_vfs.deinit(); - var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.registerBackend("https", vfs_https.backend()); - try vfs.registerBackend("hf", hf_vfs.backend()); - try vfs.registerBackend("s3", s3_vfs.backend()); - try vfs.registerBackend("gs", gcs_vfs.backend()); + var remotes: zml.io.VFS.Remotes = undefined; + try remotes.init(allocator, &vfs, init.environ_map); + defer remotes.deinit(); const io = vfs.io(); diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 83c61c90..29c67e0d 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -53,24 +53,12 @@ pub fn main(init: std.process.Init) !void { // // Virtual File Systems // - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - defer http_client.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); - defer gcs_vfs.deinit(); - var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.registerBackend("gs", gcs_vfs.backend()); - try vfs.registerBackend("hf", hf_vfs.backend()); - try vfs.registerBackend("s3", s3_vfs.backend()); + var remotes: zml.io.VFS.Remotes = undefined; + try remotes.init(allocator, &vfs, init.environ_map); + defer remotes.deinit(); const io = vfs.io(); diff --git a/vfs/vfs.zig b/vfs/vfs.zig index 930e0d6a..4ef84a9a 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -1,9 +1,9 @@ //! A `std.Io` over local files and remote objects. A bare path or a //! `file://` URI is served by the inner io; a URI such as //! `hf://org/model/model.safetensors` or `s3://bucket/key` by the backend -//! registered for its scheme (`registerBackend`). Readers take `io()` and -//! need not know which they read; one that must (to size its reads) asks -//! `fromIo`. +//! registered for its scheme (`registerBackend`, or the four of `Remotes`). +//! Readers take `io()` and need not know which they read; one that must (to +//! size its reads) asks `fromIo`. //! //! The VFS, and every backend registered on it, must outlive each file //! opened through it. @@ -86,6 +86,61 @@ pub fn registerBackend(self: *VFS, scheme: []const u8, backend: Backend) std.mem try self.backends.put(self.allocator, scheme, backend); } +/// The remote schemes on one HTTP client: `https`, `hf`, `s3` and `gs`, each +/// configured from the environment through its backend's `auto` (plain +/// `HTTP.init` for `https`), the client honoring the proxy variables. +/// Initialized in place, since the backends point at the client; it must not +/// move afterwards. Close every file opened through these schemes before +/// `deinit`; the VFS must not reach them after it. +pub const Remotes = struct { + /// The proxies' allocations, which must outlive the client. + arena: std.heap.ArenaAllocator, + client: std.http.Client, + https: HTTP, + hf: HF, + s3: S3, + gs: GCS, + + /// Builds the client on the VFS's base io and registers the four + /// schemes, replacing any backend registered under them. `environ` is + /// read here only: tokens, credentials and proxies are copied. On error + /// nothing is registered. + pub fn init(self: *Remotes, allocator: std.mem.Allocator, vfs: *VFS, environ: *std.process.Environ.Map) !void { + const base_io = vfs.base.inner; + self.arena = .init(allocator); + errdefer self.arena.deinit(); + self.client = .{ .allocator = allocator, .io = base_io }; + errdefer self.client.deinit(); + try self.client.initDefaultProxies(self.arena.allocator(), environ); + self.https = try .init(allocator, base_io, &self.client, .https); + errdefer self.https.deinit(); + self.hf = try .auto(allocator, base_io, &self.client, environ); + errdefer self.hf.deinit(); + self.s3 = try .auto(allocator, base_io, &self.client, environ); + errdefer self.s3.deinit(); + self.gs = try .auto(allocator, base_io, &self.client, environ); + errdefer self.gs.deinit(); + + vfs.mutex.lockUncancelable(base_io); + defer vfs.mutex.unlock(base_io); + // Room first: once a scheme is registered, nothing can fail. + try vfs.backends.ensureUnusedCapacity(vfs.allocator, 4); + vfs.backends.putAssumeCapacity("https", self.https.backend()); + vfs.backends.putAssumeCapacity("hf", self.hf.backend()); + vfs.backends.putAssumeCapacity("s3", self.s3.backend()); + vfs.backends.putAssumeCapacity("gs", self.gs.backend()); + } + + pub fn deinit(self: *Remotes) void { + self.gs.deinit(); + self.s3.deinit(); + self.hf.deinit(); + self.https.deinit(); + self.client.deinit(); + self.arena.deinit(); + } +}; + /// The `std.Io` of this VFS. It points into the VFS, which must not move /// once this was called. pub fn io(self: *VFS) std.Io { @@ -501,6 +556,36 @@ test "VFS gives the read hints of local and registered paths" { try std.testing.expectEqual(@as(?*VFS, null), VFS.fromIo(std.testing.io)); } +test "Remotes registers every remote scheme with its read hints" { + var filesystem = try VFS.init(std.testing.allocator, std.testing.io); + defer filesystem.deinit(); + // No credentials and no proxies: nothing is read from the user's home. + var environ: std.process.Environ.Map = .init(std.testing.allocator); + defer environ.deinit(); + var remotes: Remotes = undefined; + try remotes.init(std.testing.allocator, &filesystem, &environ); + defer remotes.deinit(); + + const Case = struct { path: []const u8, local: bool = false, hints: ReadHints }; + const remote: ReadHints = .{ .read_chunk_size = 16 * 1024 * 1024, .high_latency = true }; + const cases = [_]Case{ + .{ .path = "file:///tmp/model", .local = true, .hints = .local }, + .{ .path = "/var/models/model", .local = true, .hints = .local }, + .{ .path = "https://example.com/model", .hints = remote }, + .{ .path = "s3://bucket/model", .hints = remote }, + .{ .path = "gs://bucket/model", .hints = remote }, + .{ .path = "hf://owner/model", .hints = .{ .read_chunk_size = 32 * 1024 * 1024, .high_latency = true } }, + }; + for (cases) |case| { + try std.testing.expectEqual(case.hints, try filesystem.readHints(case.path)); + try std.testing.expectEqual(case.local, filesystem.isLocal(case.path)); + if (case.local) continue; + // A remote real path carries its scheme once, resolved without a request. + var real_path: [64]u8 = undefined; + try std.testing.expectEqualStrings(case.path, real_path[0..try std.Io.Dir.cwd().realPathFile(filesystem.io(), case.path, &real_path)]); + } +} + test "VFS serves bare paths and file:// URIs from the inner Io" { const testing_io = std.testing.io; var tmp = std.testing.tmpDir(.{}); From 490f60f7228b8e9fe19d7875861c69b095cb8c82 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:50:05 +0100 Subject: [PATCH 028/103] vfs: translate local writes and refuse remote ones Creating or deleting through the VFS went through VFSBase's forwarders, which handed the inner io a VFS handle index as a descriptor and a URI as a local path: creating a directory from a directory opened through the VFS used the wrong descriptor, and createFile on s3://bucket/key wrote bucket/key on local disk. Creates and deletes (directories, plain and atomic files) now resolve their directory through the VFS's handle table and strip file://, and the directory dirCreateDirPathOpen opens gets a handle of its own. A remote path, given whole or relative to a directory opened on a remote, is refused with ReadOnlyFileSystem before any request, and so is opening a remote file for anything but reading. The refusal is logged through base.scopedLog, which logs err at debug in test builds, so the tests of these failure paths pass under the test runner. Co-Authored-By: Claude Opus 5.5 --- vfs/base.zig | 17 ++++++ vfs/vfs.zig | 166 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 175 insertions(+), 8 deletions(-) diff --git a/vfs/base.zig b/vfs/base.zig index 6d97b584..9af3df66 100644 --- a/vfs/base.zig +++ b/vfs/base.zig @@ -1,7 +1,24 @@ const std = @import("std"); +const builtin = @import("builtin"); const stdx = @import("stdx"); +/// `std.log.scoped`, except that test builds log `err` at `debug`: the test +/// runner counts logged errors as failures, and a test of a failure path +/// checks the error it returns. +pub fn scopedLog(comptime scope: @EnumLiteral()) type { + const scoped = std.log.scoped(scope); + return struct { + pub const debug = scoped.debug; + pub const info = scoped.info; + pub const warn = scoped.warn; + + pub fn err(comptime format: []const u8, args: anytype) void { + if (builtin.is_test) scoped.debug(format, args) else scoped.err(format, args); + } + }; +} + /// How a backend's files are best read (`VFS.readHints`). No one source /// request size suits every backend: warm local 8 MiB requests on one B70 /// reached 27.05 GiB/s against 21.33 at 32 MiB, with a quarter of the pinned diff --git a/vfs/vfs.zig b/vfs/vfs.zig index 4ef84a9a..c1acdbc4 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -7,6 +7,11 @@ //! //! The VFS, and every backend registered on it, must outlive each file //! opened through it. +//! +//! A remote path is read-only: creating or deleting a file or a directory +//! there, or opening a file for writing, is logged at err and fails with +//! `ReadOnlyFileSystem`. A local file fails as the inner io does (a denied +//! one is `AccessDenied` there). const std = @import("std"); const stdx = @import("stdx"); @@ -25,6 +30,9 @@ test { } const log = std.log.scoped(.@"zml/vfs"); +/// For the refusals of writes to a remote path, which tests trigger on +/// purpose; every other err here still fails a test (`scopedLog`). +const refusal_log = base_module.scopedLog(.@"zml/vfs"); /// Indices 0-2 are the standard streams, so `std.Io.File.stdout()` and /// friends resolve to their own descriptor; the cwd placeholder follows. @@ -159,16 +167,22 @@ pub fn fromIo(any_io: std.Io) ?*VFS { const io_vtable: std.Io.VTable = VFSBase.vtable(.{ .operate = operate, + .dirCreateDir = dirCreateDir, + .dirCreateDirPath = dirCreateDirPath, + .dirCreateDirPathOpen = dirCreateDirPathOpen, .dirOpenDir = dirOpenDir, .dirStat = dirStat, .dirStatFile = dirStatFile, .dirAccess = dirAccess, .dirCreateFile = dirCreateFile, + .dirCreateFileAtomic = dirCreateFileAtomic, .dirOpenFile = dirOpenFile, .dirClose = dirClose, .dirRead = dirRead, .dirRealPath = dirRealPath, .dirRealPathFile = dirRealPathFile, + .dirDeleteFile = dirDeleteFile, + .dirDeleteDir = dirDeleteDir, .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, @@ -277,6 +291,21 @@ fn stripScheme(path: []const u8) []const u8 { return path[i + 3 ..]; } +/// The inner `Io`'s directory for creating or deleting `sub_path` from +/// `dir`. A remote path is refused: no remote backend writes, and the base +/// layer under each one would write a local file in its place. +fn writableDir(self: *VFS, dir: std.Io.Dir, sub_path: []const u8, operation: []const u8) error{ ReadOnlyFileSystem, Unexpected }!std.Io.Dir { + const backend_idx, const dir_, _ = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend to {s} '{s}' : {any}", .{ operation, sub_path, err }); + return error.Unexpected; + }; + if (self.getScheme(backend_idx)) |scheme| { + refusal_log.err("Cannot {s} '{s}': {s} paths are read-only", .{ operation, sub_path, scheme }); + return error.ReadOnlyFileSystem; + } + return dir_; +} + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); switch (operation) { @@ -302,6 +331,32 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable } } +fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create directory"); + return self.base.inner.vtable.dirCreateDir(self.base.inner.userdata, dir_, stripScheme(sub_path), permissions); +} + +fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create directory"); + return self.base.inner.vtable.dirCreateDirPath(self.base.inner.userdata, dir_, stripScheme(sub_path), permissions); +} + +fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create directory"); + const inner = self.base.inner; + const fs_dir = try inner.vtable.dirCreateDirPathOpen(inner.userdata, dir_, stripScheme(sub_path), permissions, options); + errdefer fs_dir.close(inner); + const idx, const handle = self.openHandle() catch return std.Io.Dir.CreateDirPathOpenError.Unexpected; + handle.* = .{ + .handle = @intCast(fs_dir.handle), + .backend_idx = null, + }; + return .{ .handle = @intCast(idx) }; +} + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { @@ -346,28 +401,36 @@ pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, o pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); - return std.Io.File.OpenError.Unexpected; - }; - - const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); - errdefer file.close(backend); + const dir_ = try self.writableDir(dir, sub_path, "create file"); + const inner = self.base.inner; + const file = try inner.vtable.dirCreateFile(inner.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(inner); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), - .backend_idx = backend_idx, + .backend_idx = null, .flags = file.flags, }; return .{ .handle = @intCast(idx), .flags = file.flags }; } +/// The atomic file holds the inner `Io`'s handles, not this VFS's. +fn dirCreateFileAtomic(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.CreateFileAtomicOptions) std.Io.Dir.CreateFileAtomicError!std.Io.File.Atomic { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create file"); + return self.base.inner.vtable.dirCreateFileAtomic(self.base.inner.userdata, dir_, stripScheme(sub_path), options); +} + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); return std.Io.File.OpenError.Unexpected; }; + if (flags.mode != .read_only and backend_idx != null) { + refusal_log.err("Cannot open '{s}' for writing: {s} paths are read-only", .{ sub_path, self.getScheme(backend_idx).? }); + return error.ReadOnlyFileSystem; + } const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; @@ -437,6 +500,18 @@ fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8 } } +fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "delete file"); + return self.base.inner.vtable.dirDeleteFile(self.base.inner.userdata, dir_, stripScheme(sub_path)); +} + +fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "delete directory"); + return self.base.inner.vtable.dirDeleteDir(self.base.inner.userdata, dir_, stripScheme(sub_path)); +} + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); @@ -671,6 +746,81 @@ test "copy a local file through the VFS" { try std.testing.expectEqualSlices(u8, &source_data, copied); } +test "creates and deletes on a local path go to the inner Io, from a path or an open directory" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; + var vfs: VFS = try .init(std.testing.allocator, testing_io); + defer vfs.deinit(); + const vfs_io = vfs.io(); + + var uri_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + { + const opened = try std.Io.Dir.cwd().createDirPathOpen(vfs_io, try std.fmt.bufPrint(&uri_buffer, "file://{s}/a/b", .{root}), .{}); + defer opened.close(vfs_io); + // Relative to the descriptor the VFS's handle stands for. + try opened.createDir(vfs_io, "c", .default_dir); + try opened.writeFile(vfs_io, .{ .sub_path = "c/file", .data = "bytes" }); + try tmp.dir.access(testing_io, "a/b/c/file", .{}); + try opened.deleteFile(vfs_io, "c/file"); + try opened.deleteDir(vfs_io, "c"); + // A colon without "://" is part of a local name. + try opened.writeFile(vfs_io, .{ .sub_path = "notes:old.txt", .data = "bytes" }); + try tmp.dir.access(testing_io, "a/b/notes:old.txt", .{}); + try opened.deleteFile(vfs_io, "notes:old.txt"); + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/b/notes:old.txt", .{})); + } + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/b/c", .{})); + try std.Io.Dir.cwd().createDirPath(vfs_io, try std.fmt.bufPrint(&uri_buffer, "{s}/a/d", .{root})); + try tmp.dir.access(testing_io, "a/d", .{}); + try std.Io.Dir.cwd().deleteDir(vfs_io, try std.fmt.bufPrint(&uri_buffer, "{s}/a/d", .{root})); + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/d", .{})); +} + +test "creates and deletes on a remote path fail and write nothing anywhere" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "existing", .data = "bytes" }); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; + + // The inner `Io` itself serves `s3`, rooted at the test's directory: a + // write that got past the VFS would land there. + var vfs: VFS = try .init(std.testing.allocator, testing_io); + defer vfs.deinit(); + try vfs.registerBackend("s3", .{ .io = testing_io }); + const vfs_io = vfs.io(); + const cwd = std.Io.Dir.cwd(); + + var uri_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + const uri = struct { + fn at(buffer: []u8, root_: []const u8, name: []const u8) ![]const u8 { + return std.fmt.bufPrint(buffer, "s3://{s}/{s}", .{ root_, name }); + } + }.at; + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFile(vfs_io, try uri(&uri_buffer, root, "key"), .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.openFile(vfs_io, try uri(&uri_buffer, root, "existing"), .{ .mode = .read_write })); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFileAtomic(vfs_io, try uri(&uri_buffer, root, "key"), .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDir(vfs_io, try uri(&uri_buffer, root, "dir"), .default_dir)); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPath(vfs_io, try uri(&uri_buffer, root, "a/b"))); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPathOpen(vfs_io, try uri(&uri_buffer, root, "a/b"), .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteFile(vfs_io, try uri(&uri_buffer, root, "existing"))); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteDir(vfs_io, try uri(&uri_buffer, root, "dir"))); + // A directory opened on the remote is no way around it. + const bucket = try cwd.openDir(vfs_io, try std.fmt.bufPrint(&uri_buffer, "s3://{s}", .{root}), .{}); + defer bucket.close(vfs_io); + try std.testing.expectError(error.ReadOnlyFileSystem, bucket.createFile(vfs_io, "key", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, bucket.deleteFile(vfs_io, "existing")); + + for ([_][]const u8{ "key", "dir", "a" }) |name| { + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, name, .{})); + } + try tmp.dir.access(testing_io, "existing", .{}); +} + test "a name with a colon and no \"://\" is local" { const testing_io = std.testing.io; var tmp = std.testing.tmpDir(.{}); From 89b474cf33e5d5701fd3dc3c571e3e421f755a61 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:52:59 +0100 Subject: [PATCH 029/103] vfs: add direct I/O off and on as the local case A reader that reads a local file in large aligned requests can ask the VFS to read it past the page cache: useDirectIo(file, policy) sets O_DIRECT on the descriptor under .on, once per handle, and from then on the VFS issues that handle's positional reads itself, with preadv in batches of at most IOV_MAX buffers. The kernel answers a direct read it cannot serve with EINVAL or EFAULT, which the threaded io treats as a programmer error; here such a refusal clears the flag and the read, and every later one, goes through the inner io buffered. It is logged at warn when the read was aligned and at err when it was not. Streaming reads and file copies take the handle off direct reads first. The flag goes on and off through fcntl, retried on EINTR, and only on a descriptor opened read-only. vfs/direct_io.zig holds the mechanism; the VFS exports DirectIo and direct_io_alignment (4096, or 0 where O_DIRECT does not exist). Nothing calls useDirectIo yet, so every handle stays undecided and reads as before. Co-Authored-By: Claude Opus 5.5 --- vfs/BUILD.bazel | 1 + vfs/direct_io.zig | 267 ++++++++++++++++++++++++++++++++++++++++++++++ vfs/vfs.zig | 121 +++++++++++++++++++-- 3 files changed, 381 insertions(+), 8 deletions(-) create mode 100644 vfs/direct_io.zig diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index 18f62787..6b92dab9 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -4,6 +4,7 @@ zig_library( name = "vfs", srcs = [ "base.zig", + "direct_io.zig", "gcs.zig", "hf.zig", "http.zig", diff --git a/vfs/direct_io.zig b/vfs/direct_io.zig new file mode 100644 index 00000000..7fce4d66 --- /dev/null +++ b/vfs/direct_io.zig @@ -0,0 +1,267 @@ +//! Direct I/O on the descriptor of a local file. A VFS handle's `Mode` sets +//! the flag on its descriptor when a reader asks for it under a `Policy`, +//! and from then on issues that descriptor's positional reads here rather +//! than through the inner `std.Io`: the kernel answers an unsupported direct +//! read with `EINVAL` or `EFAULT`, which the threaded `std.Io` treats as a +//! programmer error, whereas here it is a refusal that clears the flag and +//! leaves the read to the VFS, buffered. Everything below `Mode` is Linux +//! only and reached from code gated on `supported`. +//! +//! Neither path wins everywhere, and residency alone cannot predict which +//! does: warm replicated Llama-3.1-8B on eight MI300X took ~1.10 s buffered +//! but ~4.53 s forced direct from a slow storage extent; on four GB300 with +//! four NVMe drives in RAID0, direct took ~0.30 s versus ~0.32 s warm +//! buffered. These are loader times, not disk-only rates. +const std = @import("std"); +const builtin = @import("builtin"); + +const linux = std.os.linux; + +const log = std.log.scoped(.@"zml/vfs"); + +pub const supported = builtin.target.os.tag == .linux and @hasField(std.posix.O, "DIRECT"); + +/// Alignment of a direct read's offset, every buffer and the total length. +/// 4 KiB covered the measured filesystems; this is not a statx(STATX_DIOALIGN) +/// query. A filesystem requiring more alignment may reject the first read, +/// after which the VFS demotes the handle to buffered. +pub const alignment: usize = 4 * 1024; + +/// Whether a local file is read past the page cache. A direct read comes +/// from the disk even when the cache holds the data; it also leaves the cache +/// as it found it, so a file read directly is still cold for the next load. +/// Raw warm-cache reads exceeded disk throughput in the measured setups, +/// but the loader can still favor direct reads: page-cache copies compete +/// with DMA for host memory bandwidth. Conversely, repeated loads from a +/// slow disk can benefit greatly from warming the cache with `off`. The +/// contrasting GB300/MI300X load results are in this file's header. +pub const Policy = enum { + off, + /// Every file the filesystem allows. + on, +}; + +/// A local handle's read mode: undecided until a reader asks for direct +/// reads (`VFS.useDirectIo`), then buffered or direct for the rest of the +/// handle's life, since the flag belongs to the open file description and a +/// reader that was answered buffered plans exact reads that a direct +/// descriptor would reject. A direct handle goes back to buffered once the +/// kernel rejected a read, or a read went through the inner `Io`. Each +/// method takes the inner `Io` and the handle's file on it. +pub const Mode = struct { + /// Changes only under `mutex`, together with the flag; reads only load it. + state: std.atomic.Value(State) = .init(.undecided), + mutex: std.Io.Mutex = .init, + + const State = enum(u8) { undecided, buffered, direct }; + + /// Whether `file` reads direct, deciding an undecided handle under + /// `policy` (`VFS.useDirectIo`); `off` decides nothing. + pub fn decide(self: *Mode, io: std.Io, file: std.Io.File, policy: Policy) bool { + if (comptime !supported) return false; + if (policy == .off) return false; + switch (self.state.load(.acquire)) { + .direct => return true, + .buffered => return false, + .undecided => {}, + } + const stat = file.stat(io) catch return self.settle(io, file, false); + if (stat.kind != .file or stat.size < alignment) return self.settle(io, file, false); + return self.settle(io, file, true); + } + + /// Decides an undecided handle under the mutex, setting the flag first + /// when `direct`, and returns whether the handle reads direct: a + /// decision already taken (by a racing call, or a demotion) stands. A + /// descriptor the flag cannot go on (`enable`) is read buffered. + fn settle(self: *Mode, io: std.Io, file: std.Io.File, direct: bool) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + switch (self.state.load(.acquire)) { + .direct => return true, + .buffered => return false, + .undecided => {}, + } + if (direct) { + const name: Name = .{ .io = io, .file = file }; + const err = enable(file.handle); + if (err == .SUCCESS) { + self.state.store(.direct, .release); + log.debug("{f}: reading direct", .{name}); + return true; + } + log.debug("{f}: O_DIRECT refused ({t}); reading buffered", .{ name, err }); + } + self.state.store(.buffered, .release); + return false; + } + + /// The bytes read from `offset` into `data` past the page cache, or + /// null for the caller to read them buffered: the handle is not direct, + /// or the kernel refused the read and the handle is buffered from now + /// on (`logRefusal`). A refusal demotes rather than fails: a legitimate + /// short direct read can leave an unaligned continuation, and a + /// filesystem accepting the flag does not prove it accepts these + /// buffers (including PJRT-pinned memory) at read time. + pub fn read(self: *Mode, io: std.Io, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!?usize { + if (comptime !supported) return null; + if (self.state.load(.acquire) != .direct) return null; + switch (try readPositional(file.handle, data, offset)) { + .bytes => |bytes| return bytes, + .refused => |err| { + if (self.demote(io, file)) logRefusal(.{ .io = io, .file = file }, err, data, offset); + return null; + }, + } + } + + /// Takes a direct handle back to buffered before a read the VFS does + /// not issue itself (streaming, or as the source of a file copy): the + /// inner `Io` treats the errno of a rejected direct read as a + /// programmer error. + pub fn leave(self: *Mode, io: std.Io, file: std.Io.File) void { + if (comptime !supported) return; + if (self.state.load(.acquire) != .direct or !self.demote(io, file)) return; + const name: Name = .{ .io = io, .file = file }; + log.warn("{f}: buffered read of a direct file; reading it buffered from now on", .{name}); + } + + /// Takes a direct handle back to buffered: the flag comes off the + /// descriptor before the state says buffered, under the mutex, so the + /// inner `Io` never reads a descriptor with the flag on. True for the + /// call that did it, which then says why. + fn demote(self: *Mode, io: std.Io, file: std.Io.File) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + if (self.state.load(.acquire) != .direct) return false; + disable(file.handle); + self.state.store(.buffered, .release); + return true; + } +}; + +/// A local file's path in a log line, resolved only when the line is +/// written. +const Name = struct { + io: std.Io, + file: std.Io.File, + + pub fn format(self: Name, writer: *std.Io.Writer) !void { + var buffer: [std.fs.max_path_bytes]u8 = undefined; + const len = self.file.realPath(self.io, &buffer) catch return writer.writeAll(""); + try writer.writeAll(buffer[0..len]); + } +}; + +/// The kernel rejected a direct read. An aligned one is the filesystem +/// refusing direct I/O it accepted the flag for, a warning; a misaligned +/// one is the reader's, either its plan or a continuation after a short +/// read (which a network filesystem may answer), an error. Either way the +/// file is read buffered from now on. +fn logRefusal(name: Name, err: linux.E, data: []const []u8, offset: u64) void { + var total: usize = 0; + var aligned = offset % alignment == 0; + for (data) |buffer| { + if (buffer.len == 0) continue; + aligned = aligned and @intFromPtr(buffer.ptr) % alignment == 0; + total += buffer.len; + } + aligned = aligned and total % alignment == 0; + if (aligned) { + log.warn("{f}: direct read refused ({t}) at offset={d} total={d}; reading the file buffered from now on", .{ name, err, offset, total }); + } else { + log.err("{f}: misaligned read of a direct file ({t}) at offset={d} first_buffer=0x{x} total={d} alignment={d}; reading the file buffered from now on", .{ + name, + err, + offset, + if (data.len != 0) @intFromPtr(data[0].ptr) else 0, + total, + alignment, + }); + } +} + +const max_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 else std.posix.IOV_MAX; +const direct_flag: usize = @as(u32, @bitCast(std.posix.O{ .DIRECT = true })); + +/// Sets the flag on a descriptor opened for reading only, since the flag +/// constrains writes too: one opened for writing is refused with `BADF`. A +/// filesystem without direct I/O refuses the flag. +pub fn enable(fd: std.posix.fd_t) linux.E { + const current = switch (getStatusFlags(fd)) { + .flags => |flags| flags, + .err => |err| return err, + }; + const status: std.posix.O = @bitCast(@as(u32, @truncate(current))); + if (status.ACCMODE != .RDONLY) return .BADF; + return setStatusFlags(fd, current | direct_flag); +} + +/// Clears the flag, so the inner `Io` may read the descriptor again. +fn disable(fd: std.posix.fd_t) void { + const current = switch (getStatusFlags(fd)) { + .flags => |flags| flags, + .err => return, + }; + _ = setStatusFlags(fd, current & ~direct_flag); +} + +/// `fcntl(F_GETFL)`, retried on `EINTR`. +fn getStatusFlags(fd: std.posix.fd_t) union(enum) { flags: usize, err: linux.E } { + while (true) { + const rc = linux.fcntl(fd, linux.F.GETFL, 0); + switch (linux.errno(rc)) { + .SUCCESS => return .{ .flags = rc }, + .INTR => continue, + else => |err| return .{ .err = err }, + } + } +} + +/// `fcntl(F_SETFL)`, retried on `EINTR`. +fn setStatusFlags(fd: std.posix.fd_t, flags: usize) linux.E { + while (true) { + switch (linux.errno(linux.fcntl(fd, linux.F.SETFL, flags))) { + .INTR => continue, + else => |err| return err, + } + } +} + +const ReadResult = union(enum) { + bytes: usize, + /// The kernel rejected the direct read: an unaligned offset, buffer or + /// total, a filesystem that took the flag but not the read, or memory + /// the block layer cannot pin. + refused: linux.E, +}; + +/// Reads `data` at `offset` on a descriptor with the flag set, at most +/// `IOV_MAX` buffers per call (the caller loops). A read that runs past the +/// end of the file is cut there. +fn readPositional(fd: std.posix.fd_t, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!ReadResult { + var iovecs: [max_iovecs]std.posix.iovec = undefined; + var count: usize = 0; + for (data) |buffer| { + if (buffer.len == 0) continue; + if (count == iovecs.len) break; + iovecs[count] = .{ .base = buffer.ptr, .len = buffer.len }; + count += 1; + } + if (count == 0) return .{ .bytes = 0 }; + while (true) { + const rc = linux.preadv(fd, &iovecs, count, @intCast(offset)); + switch (linux.errno(rc)) { + .SUCCESS => return .{ .bytes = rc }, + .INTR => continue, + .INVAL, .FAULT, .OPNOTSUPP => |err| return .{ .refused = err }, + .IO => return error.InputOutput, + .NOMEM => return error.SystemResources, + .AGAIN => return error.WouldBlock, + .ISDIR => return error.IsDir, + .BADF => return error.NotOpenForReading, + .NXIO, .SPIPE => return error.Unseekable, + else => |err| return std.posix.unexpectedErrno(err), + } + } +} diff --git a/vfs/vfs.zig b/vfs/vfs.zig index c1acdbc4..0b0c2f8c 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -2,8 +2,8 @@ //! `file://` URI is served by the inner io; a URI such as //! `hf://org/model/model.safetensors` or `s3://bucket/key` by the backend //! registered for its scheme (`registerBackend`, or the four of `Remotes`). -//! Readers take `io()` and need not know which they read; one that must (to -//! size its reads) asks `fromIo`. +//! Readers take `io()` and need not know which they read; one that must (for +//! read sizes and direct I/O) asks `fromIo`. //! //! The VFS, and every backend registered on it, must outlive each file //! opened through it. @@ -11,7 +11,8 @@ //! A remote path is read-only: creating or deleting a file or a directory //! there, or opening a file for writing, is logged at err and fails with //! `ReadOnlyFileSystem`. A local file fails as the inner io does (a denied -//! one is `AccessDenied` there). +//! one is `AccessDenied` there); it may be read past the page cache +//! (`useDirectIo`). const std = @import("std"); const stdx = @import("stdx"); @@ -21,9 +22,15 @@ pub const HF = @import("hf.zig").HF; pub const HTTP = @import("http.zig").HTTP; pub const S3 = @import("s3.zig").S3; const base_module = @import("base.zig"); +const direct_io = @import("direct_io.zig"); pub const Backend = base_module.Backend; pub const ReadHints = base_module.ReadHints; const VFSBase = base_module.VFSBase; +/// See `useDirectIo`. +pub const DirectIo = direct_io.Policy; +/// What every read of a direct file must meet in offset, buffers and total +/// (`useDirectIo`); 0 on a system without direct I/O, where no file is. +pub const direct_io_alignment: usize = if (direct_io.supported) direct_io.alignment else 0; test { _ = @import("gcs.zig"); @@ -46,6 +53,8 @@ const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false }, + /// A local file's read mode (`useDirectIo`). + direct: direct_io.Mode = .{}, fn innerFile(self: *const Handle) std.Io.File { return .{ .handle = @intCast(self.handle), .flags = self.flags }; @@ -205,8 +214,8 @@ pub fn readHints(self: *VFS, path: []const u8) !ReadHints { return self.backends.entries.items(.value)[index].read_hints; } -/// Whether `path` resolves to the inner `Io`'s files; an unregistered scheme -/// does not. Opens nothing. +/// Whether `path` resolves to a local file, the only kind `useDirectIo` can +/// read directly; an unregistered scheme is not. Opens nothing. pub fn isLocal(self: *VFS, path: []const u8) bool { const backend_idx, _, _ = self.lookupDir(.cwd(), path) catch return false; return backend_idx == null; @@ -239,6 +248,30 @@ fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { return .{ handle, self.getBackend(handle.backend_idx) }; } +/// Direct reads for a local file under `policy`: sets the flag on a +/// read-only descriptor of a regular file at least one alignment unit long +/// and returns true, after which every read of the file must meet +/// `direct_io_alignment` in offset, buffers and total. False leaves the file +/// buffered: another backend's file, `off`, a filesystem that refuses the +/// flag, or a file whose direct read the kernel already rejected. The first +/// call under `on` decides for the handle's life (`direct_io.Mode`); the +/// caller must ask before it reads. The mechanism lives in the VFS because +/// it owns the descriptor; one descriptor suffices because a reader reads a +/// file in one mode. +pub fn useDirectIo(self: *VFS, file: std.Io.File, policy: DirectIo) bool { + const handle, _ = self.getFileHandle(file); + if (handle.backend_idx != null) return false; + return handle.direct.decide(self.base.inner, handle.innerFile(), policy); +} + +/// A local file as the inner `Io` may read it, off direct reads +/// (`direct_io.Mode.leave`). +fn innerReadable(self: *VFS, handle: *Handle) std.Io.File { + const file = handle.innerFile(); + handle.direct.leave(self.base.inner, file); + return file; +} + fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -312,7 +345,7 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable .file_read_streaming => |o| { const handle, const backend = self.getFileHandle(o.file); return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ - .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .file = self.innerReadable(handle), .data = o.data, } }); }, @@ -551,7 +584,7 @@ fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []co const original_src_io = reader.io; const original_src_handle = reader.file; reader.io = src_backend; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + reader.file = self.innerReadable(src_handle); defer { reader.io = original_src_io; reader.file = original_src_handle; @@ -572,7 +605,7 @@ fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []c const original_src_io = reader.io; const original_src_handle = reader.file; reader.io = src_backend; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + reader.file = self.innerReadable(src_handle); defer { reader.io = original_src_io; reader.file = original_src_handle; @@ -584,6 +617,7 @@ fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []c fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); + if (try handle.direct.read(self.base.inner, handle.innerFile(), data, offset)) |bytes| return bytes; return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); } @@ -653,6 +687,7 @@ test "Remotes registers every remote scheme with its read hints" { }; for (cases) |case| { try std.testing.expectEqual(case.hints, try filesystem.readHints(case.path)); + // Only a local file can be read directly; nothing is opened to ask. try std.testing.expectEqual(case.local, filesystem.isLocal(case.path)); if (case.local) continue; // A remote real path carries its scheme once, resolved without a request. @@ -684,12 +719,82 @@ test "VFS serves bare paths and file:// URIs from the inner Io" { defer opened.close(vfs_io); const handle, _ = filesystem.getFileHandle(opened); try std.testing.expectEqual(@as(?usize, null), handle.backend_idx); + // A file this small is never direct. + try std.testing.expect(!filesystem.useDirectIo(opened, .on)); var contents: [15]u8 = undefined; try std.testing.expectEqual(contents.len, try opened.readPositionalAll(vfs_io, &contents, 0)); try std.testing.expectEqualStrings("bare path bytes", &contents); } } +test "VFS reads the same bytes under every direct I/O policy" { + const testing_io = std.testing.io; + const alignment = 4096; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var contents: [3 * alignment + 100]u8 = undefined; + for (&contents, 0..) |*byte, i| byte.* = @truncate(i * 7 + 3); + const created = try tmp.dir.createFile(testing_io, "data.bin", .{ .read = true }); + try created.writePositionalAll(testing_io, &contents, 0); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; + created.close(testing_io); + + var filesystem = try VFS.init(std.testing.allocator, testing_io); + defer filesystem.deinit(); + const vfs_io = filesystem.io(); + const buffer = try std.testing.allocator.alignedAlloc(u8, .fromByteUnits(alignment), 2 * alignment); + defer std.testing.allocator.free(buffer); + const plain: []u8 = buffer; + for ([_]DirectIo{ .on, .off }) |policy| { + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); + defer file.close(vfs_io); + // Direct when the policy and the filesystem allow it, buffered + // otherwise (`off`). The bytes are the same either way. + const direct = filesystem.useDirectIo(file, policy); + if (policy != .on) try std.testing.expect(!direct); + // The decision stands; `off` neither asks nor changes it. + try std.testing.expect(!filesystem.useDirectIo(file, .off)); + try std.testing.expectEqual(direct, filesystem.useDirectIo(file, policy)); + + // Aligned offset, buffer and length. + @memset(buffer, 0); + try std.testing.expectEqual(buffer.len, try file.readPositionalAll(vfs_io, buffer, alignment)); + try std.testing.expectEqualSlices(u8, contents[alignment..][0..buffer.len], buffer); + + // One aligned call that runs past the end of the file is cut there. + @memset(buffer, 0); + const past_end = try file.readPositional(vfs_io, &.{plain}, 2 * alignment); + try std.testing.expectEqual(contents.len - 2 * alignment, past_end); + try std.testing.expectEqualSlices(u8, contents[2 * alignment ..], buffer[0..past_end]); + try std.testing.expectEqual(direct, filesystem.useDirectIo(file, policy)); + + // A streaming read takes the file off direct reads for good. (A + // misaligned positional read of a direct file does too, with an + // error logged, which the test runner would count; a filesystem + // such as tmpfs may serve it instead, so it is not exercised here.) + var head: [8]u8 = undefined; + const head_slice: []u8 = &head; + try std.testing.expectEqual(head.len, try file.readStreaming(vfs_io, &.{head_slice})); + try std.testing.expectEqualSlices(u8, contents[0..head.len], &head); + try std.testing.expect(!filesystem.useDirectIo(file, policy)); + + // Buffered from now on: unaligned reads and the end of the file. + var tail: [50]u8 = undefined; + try std.testing.expectEqual(tail.len, try file.readPositionalAll(vfs_io, &tail, alignment + 3)); + try std.testing.expectEqualSlices(u8, contents[alignment + 3 ..][0..tail.len], &tail); + try std.testing.expectEqual(@as(usize, 0), try file.readPositional(vfs_io, &.{plain}, contents.len)); + } + + // A descriptor opened for writing is never direct: the flag would + // constrain its writes too. + const writable = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_write }); + defer writable.close(vfs_io); + try std.testing.expect(!filesystem.useDirectIo(writable, .on)); + const writable_handle, _ = filesystem.getFileHandle(writable); + if (comptime direct_io.supported) try std.testing.expectEqual(std.os.linux.E.BADF, direct_io.enable(writable_handle.innerFile().handle)); +} + test "standard streams resolve to their own descriptors" { var vfs: VFS = try .init(std.testing.allocator, std.testing.io); defer vfs.deinit(); From 77f382dfc27a3793a1e0a4039c4558ec7dfe0b89 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:54:57 +0100 Subject: [PATCH 030/103] vfs: decide direct I/O per file from page-cache residency Neither read mode wins everywhere: a direct read comes from the disk even when the page cache holds the file, and a buffered one pays a copy that competes with DMA for host memory bandwidth. DirectIo.auto reads a file directly when at most half of 32 pages sampled across it are cached, and buffered otherwise. A sample is a preadv2 with RWF_NOWAIT, which answers EAGAIN for an uncached page, needs no write permission and costs the same whatever the file's size; unknown residency counts as cached. A sample that misses queues the read of its page, so each probe shifts its samples by a random offset, and a cold file read directly is still found cold by the next probe unless its samples land in what those misses read. predictDirectIo gives the answer useDirectIo would give for a local path, and the file's size, on a descriptor of its own: it decides no handle and logs nothing. Co-Authored-By: Claude Opus 5.5 --- vfs/direct_io.zig | 103 ++++++++++++++++++++++++++++++- vfs/vfs.zig | 154 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 246 insertions(+), 11 deletions(-) diff --git a/vfs/direct_io.zig b/vfs/direct_io.zig index 7fce4d66..4d758589 100644 --- a/vfs/direct_io.zig +++ b/vfs/direct_io.zig @@ -27,9 +27,15 @@ pub const supported = builtin.target.os.tag == .linux and @hasField(std.posix.O, /// after which the VFS demotes the handle to buffered. pub const alignment: usize = 4 * 1024; +/// `Policy.auto` reads a file directly when at most this fraction of its +/// sampled pages is cached. +const cold_fraction: f64 = 0.5; + /// Whether a local file is read past the page cache. A direct read comes /// from the disk even when the cache holds the data; it also leaves the cache -/// as it found it, so a file read directly is still cold for the next load. +/// as it found it, so a file read directly is still cold for the next load +/// (the `auto` probe caches what it samples, elsewhere on every probe: +/// `cachedFraction`). /// Raw warm-cache reads exceeded disk throughput in the measured setups, /// but the loader can still favor direct reads: page-cache copies compete /// with DMA for host memory bandwidth. Conversely, repeated loads from a @@ -39,6 +45,12 @@ pub const Policy = enum { off, /// Every file the filesystem allows. on, + /// A file mostly out of the page cache when asked, buffered for a cached + /// one. Residency is sampled: `residency_samples` pages spread over the + /// file, read with `RWF_NOWAIT`, which answers `EAGAIN` for a page that + /// is not cached, needs no permission on the file and costs the same + /// whatever its size. Unknown counts as cached. + auto, }; /// A local handle's read mode: undecided until a reader asks for direct @@ -66,7 +78,19 @@ pub const Mode = struct { .undecided => {}, } const stat = file.stat(io) catch return self.settle(io, file, false); - if (stat.kind != .file or stat.size < alignment) return self.settle(io, file, false); + if (!fits(stat)) return self.settle(io, file, false); + if (policy == .auto) { + const name: Name = .{ .io = io, .file = file }; + const cached = sampleResidency(io, file, stat.size) orelse { + log.debug("{f}: page cache residency unknown; reading buffered", .{name}); + return self.settle(io, file, false); + }; + if (cached > cold_fraction) { + log.debug("{f}: {d:.0}% cached; reading buffered", .{ name, cached * 100 }); + return self.settle(io, file, false); + } + log.debug("{f}: {d:.0}% cached", .{ name, cached * 100 }); + } return self.settle(io, file, true); } @@ -140,6 +164,39 @@ pub const Mode = struct { } }; +/// A file's size, and whether `Mode.decide` would read it direct +/// (`predict`). +pub const Prediction = struct { size: u64, direct: bool }; + +/// `Mode.decide`'s answer for `file` under `policy`, deciding and logging +/// nothing: the same rule, then the flag set on `file` and left set, so +/// `file` must be a read-only descriptor of the caller's own, which the +/// caller closes. Under `auto` the sample caches what it misses, as +/// `decide`'s does: a missed sample queues the read of its page and of the +/// kernel's readahead. A decision that follows and samples those pages +/// once the reads have landed finds the file cached and reads it buffered, +/// which may happen to any file, the likelier the fewer pages it has per +/// sample (`cachedFraction`). Null when `file` cannot be stat-ed. +pub fn predict(io: std.Io, file: std.Io.File, policy: Policy) error{Canceled}!?Prediction { + if (comptime !supported) return null; + const stat = file.stat(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return null, + }; + const direct = fits(stat) and switch (policy) { + .off => false, + .on => true, + .auto => if (sampleResidency(io, file, stat.size)) |cached| cached <= cold_fraction else false, + }; + return .{ .size = stat.size, .direct = direct and enable(file.handle) == .SUCCESS }; +} + +/// A regular file at least one unit long: the only kind read direct +/// (`Mode.decide`, `predict`). +fn fits(stat: std.Io.File.Stat) bool { + return stat.kind == .file and stat.size >= alignment; +} + /// A local file's path in a log line, resolved only when the line is /// written. const Name = struct { @@ -228,6 +285,48 @@ fn setStatusFlags(fd: std.posix.fd_t, flags: usize) linux.E { } } +/// A fixed count bounds the probe's cost whatever the checkpoint's size and +/// suffices for the coarse half-cached decision. A whole-file `cachestat` +/// walks every cached folio, about 7 ms per cached GiB on a 4 KiB-page +/// kernel, paid on the planner thread, and refuses a file the process can +/// read but not write. +pub const residency_samples = 32; + +/// The fraction of `residency_samples` pages spread over the file that the +/// page cache holds, or null when the filesystem cannot say (no +/// `RWF_NOWAIT`). One sample per window of the file, each `shift` bytes +/// into its window (`shift` modulo the window), unknown residency selecting +/// buffered IO. A sample that misses queues the read of its page, which a +/// direct file never reads back: with the same offsets the next probe would +/// find every sample cached and read a cold file buffered, so callers draw a +/// new `shift` for every probe. A file of at most `residency_samples` pages +/// has every page sampled whatever the shift, so its first probe caches it. +/// The header page a parser has just read is sampled only when `shift` +/// lands in it, not systematically. +fn cachedFraction(fd: std.posix.fd_t, size: u64, shift: u64) ?f64 { + var buffer: [512]u8 = undefined; + const iovec: std.posix.iovec = .{ .base = &buffer, .len = buffer.len }; + var cached: usize = 0; + const window = size / residency_samples; + for (0..residency_samples) |sample| { + const offset = window * sample + shift % @max(window, 1); + const rc = linux.preadv2(fd, @ptrCast(&iovec), 1, @intCast(offset), linux.RWF.NOWAIT); + switch (linux.errno(rc)) { + .SUCCESS => cached += @intFromBool(rc != 0), + .AGAIN => {}, + else => return null, + } + } + return @as(f64, @floatFromInt(cached)) / @as(f64, @floatFromInt(residency_samples)); +} + +/// `cachedFraction` at a fresh shift, as every probe draws one. +fn sampleResidency(io: std.Io, file: std.Io.File, size: u64) ?f64 { + var shift: u64 = undefined; + io.random(std.mem.asBytes(&shift)); + return cachedFraction(file.handle, size, shift); +} + const ReadResult = union(enum) { bytes: usize, /// The kernel rejected the direct read: an unaligned offset, buffer or diff --git a/vfs/vfs.zig b/vfs/vfs.zig index 0b0c2f8c..c3211657 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -250,20 +250,43 @@ fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { /// Direct reads for a local file under `policy`: sets the flag on a /// read-only descriptor of a regular file at least one alignment unit long -/// and returns true, after which every read of the file must meet -/// `direct_io_alignment` in offset, buffers and total. False leaves the file -/// buffered: another backend's file, `off`, a filesystem that refuses the +/// (and, under `auto`, mostly out of the page cache) and returns true, after +/// which every read of the file must meet `direct_io_alignment` in offset, +/// buffers and total. False leaves the file buffered: another backend's +/// file, `off`, a cached file under `auto`, a filesystem that refuses the /// flag, or a file whose direct read the kernel already rejected. The first -/// call under `on` decides for the handle's life (`direct_io.Mode`); the -/// caller must ask before it reads. The mechanism lives in the VFS because -/// it owns the descriptor; one descriptor suffices because a reader reads a -/// file in one mode. +/// call under `on` or `auto` decides for the handle's life +/// (`direct_io.Mode`); the caller must ask before it reads. The mechanism +/// lives in the VFS because it owns the descriptor; one descriptor suffices +/// because a reader reads a file in one mode. pub fn useDirectIo(self: *VFS, file: std.Io.File, policy: DirectIo) bool { const handle, _ = self.getFileHandle(file); if (handle.backend_idx != null) return false; return handle.direct.decide(self.base.inner, handle.innerFile(), policy); } +/// See `predictDirectIo`. +pub const DirectIoPrediction = direct_io.Prediction; + +/// What `useDirectIo` would answer under `policy` for the local file +/// `sub_path` of `dir`, and its size, asked on a descriptor of its own that +/// is closed again: no handle is decided and nothing is logged. For a reader +/// that places memory before it plans its files. Under `auto` this sample +/// caches what it misses, so a decision that follows may find any file +/// cached and read it buffered, the likelier the fewer pages it has per +/// sample (`direct_io.predict`). Null for a path that is not local, or a +/// file that does not open or stat. +pub fn predictDirectIo(self: *VFS, dir: std.Io.Dir, sub_path: []const u8, policy: DirectIo) error{Canceled}!?DirectIoPrediction { + const backend_idx, const inner_dir, const inner = self.lookupDir(dir, sub_path) catch return null; + if (backend_idx != null) return null; + const file = inner_dir.openFile(inner, stripScheme(sub_path), .{ .mode = .read_only }) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return null, + }; + defer file.close(inner); + return direct_io.predict(inner, file, policy); +} + /// A local file as the inner `Io` may read it, off direct reads /// (`direct_io.Mode.leave`). fn innerReadable(self: *VFS, handle: *Handle) std.Io.File { @@ -746,11 +769,12 @@ test "VFS reads the same bytes under every direct I/O policy" { const buffer = try std.testing.allocator.alignedAlloc(u8, .fromByteUnits(alignment), 2 * alignment); defer std.testing.allocator.free(buffer); const plain: []u8 = buffer; - for ([_]DirectIo{ .on, .off }) |policy| { + for ([_]DirectIo{ .on, .auto, .off }) |policy| { const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); defer file.close(vfs_io); // Direct when the policy and the filesystem allow it, buffered - // otherwise (`off`). The bytes are the same either way. + // otherwise: `off`, or under `auto` a file just written and thus + // cached. The bytes are the same either way. const direct = filesystem.useDirectIo(file, policy); if (policy != .on) try std.testing.expect(!direct); // The decision stands; `off` neither asks nor changes it. @@ -795,6 +819,118 @@ test "VFS reads the same bytes under every direct I/O policy" { if (comptime direct_io.supported) try std.testing.expectEqual(std.os.linux.E.BADF, direct_io.enable(writable_handle.innerFile().handle)); } +test "predictDirectIo answers as useDirectIo would and decides no handle" { + if (comptime !direct_io.supported) return error.SkipZigTest; + const testing_io = std.testing.io; + const alignment = 4096; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents: [3 * alignment + 100]u8 = @splat(0x5a); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "data.bin", .data = &contents }); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "short.bin", .data = contents[0..100] }); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(testing_io, &root_buffer)]; + + var filesystem = try VFS.init(std.testing.allocator, testing_io); + defer filesystem.deinit(); + // Another backend's scheme, served here by the plain io: its paths open, + // but they are not local. + try filesystem.registerBackend("remote", .{ .io = testing_io }); + const vfs_io = filesystem.io(); + const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); + defer dir.close(vfs_io); + var path_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, "{s}/data.bin", .{root}); + + for ([_]DirectIo{ .on, .auto, .off }) |policy| { + // No handle is opened or decided to answer. + const handles = filesystem.handles.len; + const closed_handles = filesystem.closed_handles.items.len; + const predicted = (try filesystem.predictDirectIo(.cwd(), path, policy)).?; + try std.testing.expectEqual(@as(u64, contents.len), predicted.size); + try std.testing.expectEqual(predicted, (try filesystem.predictDirectIo(dir, "data.bin", policy)).?); + try std.testing.expectEqual(handles, filesystem.handles.len); + try std.testing.expectEqual(closed_handles, filesystem.closed_handles.items.len); + + // A fresh handle decides as predicted: direct only under `on`, where + // the filesystem allows it, since the file was just written and so + // is cached. + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); + defer file.close(vfs_io); + try std.testing.expectEqual(predicted.direct, filesystem.useDirectIo(file, policy)); + if (policy != .on) try std.testing.expect(!predicted.direct); + } + + // A file shorter than a unit is never direct. + const short = (try filesystem.predictDirectIo(dir, "short.bin", .on)).?; + try std.testing.expectEqual(DirectIoPrediction{ .size = 100, .direct = false }, short); + // Nothing to answer for a remote path, though its backend opens the same + // file, or for a missing file. + var remote_buffer: [std.fs.max_path_bytes + 32]u8 = undefined; + const remote = try std.fmt.bufPrint(&remote_buffer, "remote://{s}/data.bin", .{root}); + try std.testing.expectEqual(null, try filesystem.predictDirectIo(.cwd(), remote, .on)); + try std.testing.expectEqual(null, try filesystem.predictDirectIo(dir, "missing.bin", .on)); +} + +test "auto keeps reading a cold file direct after probing it" { + if (comptime !direct_io.supported) return error.SkipZigTest; + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // One 2 MiB window per sample (`direct_io.residency_samples`): many + // pages each, on 64 KiB-page kernels too. + const window = 2 << 20; + const contents = try std.testing.allocator.alloc(u8, direct_io.residency_samples * window); + defer std.testing.allocator.free(contents); + @memset(contents, 0x5a); + const created = try tmp.dir.createFile(testing_io, "cold.bin", .{ .read = true }); + try created.writePositionalAll(testing_io, contents, 0); + try created.sync(testing_io); + _ = std.os.linux.fadvise(created.handle, 0, 0, std.os.linux.POSIX_FADV.DONTNEED); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; + created.close(testing_io); + + // The testing io, but the probes draw shifts a quarter window apart, + // 512 KiB from each other's samples. At random shifts a probe may sample + // the pages an earlier one's misses read, and find the file cached + // (`direct_io.predict`): about once in window / page pairs of probes. + const Shifts = struct { + var drawn: u64 = 0; + + fn random(_: ?*anyopaque, buffer: []u8) void { + std.mem.writeInt(u64, buffer[0..8], window / 8 + drawn * window / 4, .native); + drawn += 1; + } + }; + Shifts.drawn = 0; + var vtable = testing_io.vtable.*; + vtable.random = Shifts.random; + const shifting_io: std.Io = .{ .userdata = testing_io.userdata, .vtable = &vtable }; + + var filesystem = try VFS.init(std.testing.allocator, shifting_io); + defer filesystem.deinit(); + const vfs_io = filesystem.io(); + for (0..2) |open| { + // Predicted first, as a reader placing memory before it plans. + const predicted = (try filesystem.predictDirectIo(.cwd(), path, .auto)).?; + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); + defer file.close(vfs_io); + const direct = filesystem.useDirectIo(file, .auto); + // A filesystem that kept the file cached (tmpfs) or refused direct + // reads says nothing here. + if (open == 0 and !direct) return error.SkipZigTest; + try std.testing.expect(predicted.direct); + // The first probes' misses queued reads of the pages they sampled; + // once they landed, sampling the same pages would find the file + // cached. + if (open == 0) try testing_io.sleep(.fromMilliseconds(250), .awake); + try std.testing.expect(direct); + } + // Each probe drew a shift of its own. + try std.testing.expectEqual(@as(u64, 4), Shifts.drawn); +} + test "standard streams resolve to their own descriptors" { var vfs: VFS = try .init(std.testing.allocator, std.testing.io); defer vfs.deinit(); From 6a155f7907b57a328c3c3945d8d29cba7b67b2d7 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:56:27 +0100 Subject: [PATCH 031/103] stdx: add FirstError, the first error of concurrent tasks Tasks that run concurrently and may each fail often need to keep the first failure and let the others see it. FirstError(E) keeps it in one atomic word: set stores an error unless one is already kept and says whether it kept this one, and get reads it back. Co-Authored-By: Claude Opus 5.5 --- stdx/stdx.zig | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 1b19e638..5ed0b048 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -87,6 +87,33 @@ pub fn Once(comptime f: anytype) type { }; } +/// The first error of concurrent tasks, kept once and readable by any of +/// them. The state is the error's integer value, zero while none is kept. +pub fn FirstError(comptime E: type) type { + return struct { + code: std.atomic.Value(@Int(.unsigned, @bitSizeOf(anyerror))) = .init(0), + + /// Keeps `err` unless an error is kept already; true when this call + /// kept it. + pub fn set(self: *@This(), err: E) bool { + return self.code.cmpxchgStrong(0, @intFromError(err), .release, .monotonic) == null; + } + + pub fn get(self: *const @This()) ?E { + const code = self.code.load(.acquire); + return if (code == 0) null else @errorCast(@errorFromInt(code)); + } + }; +} + +test FirstError { + var first: FirstError(error{ A, B }) = .{}; + try std.testing.expectEqual(null, first.get()); + try std.testing.expect(first.set(error.B)); + try std.testing.expect(!first.set(error.A)); + try std.testing.expectEqual(error.B, first.get().?); +} + /// Packed storage of sub-byte values /// /// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, From 93852470f42d513f3214a66f2499620656dda008 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 11:56:51 +0100 Subject: [PATCH 032/103] vfs: add MockServer, a loopback HTTP origin for tests The remote backends' failure paths (retries, refusals, redirects, rate limiting) cannot be tested against a real origin. MockServer serves an object per path over a real loopback socket, with Range GETs and HEADs, and can be told to fail the first GETs, hold GETs at a barrier until enough arrive, throttle (the first requests, every GET, or past a rate window, with a status and a Retry-After), refuse a path with a status, or redirect it. It counts HEADs, GETs and the peak of concurrent GETs, and keeps the first error it met through stdx.FirstError. It is exported as VFS.MockServer so tests outside vfs can use it too. Co-Authored-By: Claude Opus 5.5 --- vfs/BUILD.bazel | 1 + vfs/mock_server.zig | 326 ++++++++++++++++++++++++++++++++++++++++++++ vfs/vfs.zig | 2 + 3 files changed, 329 insertions(+) create mode 100644 vfs/mock_server.zig diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index 6b92dab9..d846af65 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -8,6 +8,7 @@ zig_library( "gcs.zig", "hf.zig", "http.zig", + "mock_server.zig", "parallel_read.zig", "s3.zig", ], diff --git a/vfs/mock_server.zig b/vfs/mock_server.zig new file mode 100644 index 00000000..9206f65b --- /dev/null +++ b/vfs/mock_server.zig @@ -0,0 +1,326 @@ +//! A minimal HTTP origin for the VFS and loader tests: one object per path, +//! range reads, and injectable failures and rate limiting. Exported as +//! `VFS.MockServer` so tests above this layer can serve a fixture over a +//! real socket; nothing outside a test build references it. + +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const MockServer = struct { + /// How the server rate limits. A throttled answer carries the status + /// and, when the test names one, a `Retry-After`. + const Throttle = struct { + /// The first N GETs are throttled. + first_gets: usize = 0, + /// The first N HEADs are throttled. + first_heads: usize = 0, + /// Every GET is throttled, forever. + always: bool = false, + status: std.http.Status = .too_many_requests, + retry_after_s: ?u32 = null, + /// At most `gets` GETs per `per_ms` milliseconds; the rest are + /// throttled. + window: ?struct { gets: usize, per_ms: i64 } = null, + }; + + /// One extra path the server serves beside `/object`. + const Object = struct { path: []const u8, bytes: []const u8 }; + + /// A path answered with `status` and no object, on purpose. + pub const Refusal = struct { path: []const u8, status: std.http.Status }; + + /// A path answered with a 302 to `location`, for HEAD and GET alike. + pub const Redirect = struct { path: []const u8, location: []const u8 }; + + pub const Options = struct { + fail_first_gets: usize = 0, + barrier_gets: usize = 0, + throttle: Throttle = .{}, + objects: []const Object = &.{}, + /// Paths answered with a status of their own (404, 403, ...). + refusals: []const Refusal = &.{}, + redirects: []const Redirect = &.{}, + }; + + tcp: std.Io.net.Server, + /// The accept loop and its connections, from `start` to `stop`. + group: std.Io.Group = .init, + object: []const u8, + fail_first_gets: usize, + barrier_gets: usize, + throttle: Throttle, + objects: []const Object, + refusals: []const Refusal, + redirects: []const Redirect, + get_barrier: std.Io.Event = .unset, + head_requests: std.atomic.Value(usize) = .init(0), + get_requests: std.atomic.Value(usize) = .init(0), + active_gets: std.atomic.Value(usize) = .init(0), + peak_gets: std.atomic.Value(usize) = .init(0), + first_error: stdx.FirstError(anyerror) = .{}, + /// Awake nanoseconds of the first throttled answer and of the first + /// answer served after one; zero until they happen. + first_throttle_ns: std.atomic.Value(i64) = .init(0), + first_get_after_throttle_ns: std.atomic.Value(i64) = .init(0), + /// Guards the rate window. + window_mutex: std.Io.Mutex = .init, + window_started_ns: i64 = 0, + window_gets: usize = 0, + + pub fn init( + io: std.Io, + object: []const u8, + opts: Options, + ) !MockServer { + const address: std.Io.net.IpAddress = .{ .ip4 = .loopback(0) }; + return .{ + .tcp = try address.listen(io, .{ .reuse_address = true }), + .object = object, + .fail_first_gets = opts.fail_first_gets, + .barrier_gets = opts.barrier_gets, + .throttle = opts.throttle, + .objects = opts.objects, + .refusals = opts.refusals, + .redirects = opts.redirects, + }; + } + + /// Serves on a task of its own until `stop`. The server must not move + /// from here on. + pub fn start(self: *MockServer, io: std.Io) !void { + try self.group.concurrent(io, run, .{ self, io }); + } + + /// Stops serving, each connection canceled and joined, and returns the + /// first error the server met. + pub fn stop(self: *MockServer, io: std.Io) !void { + self.group.cancel(io); + if (self.first_error.get()) |err| return err; + } + + /// Stops serving if `stop` did not, and closes the listening socket. + pub fn deinit(self: *MockServer, io: std.Io) void { + self.group.cancel(io); + self.tcp.deinit(io); + } + + /// The path of `/object` for an `http` backend: `127.0.0.1:/object`. + pub fn path(self: *const MockServer, buffer: []u8) ![]const u8 { + return std.fmt.bufPrint(buffer, "127.0.0.1:{d}/object", .{self.port()}); + } + + fn bodyFor(self: *const MockServer, target: []const u8) ?[]const u8 { + if (std.mem.eql(u8, target, "/object")) return self.object; + for (self.objects) |object| { + if (std.mem.eql(u8, object.path, target)) return object.bytes; + } + return null; + } + + fn refusalFor(self: *const MockServer, target: []const u8) ?std.http.Status { + for (self.refusals) |refusal| { + if (std.mem.eql(u8, refusal.path, target)) return refusal.status; + } + return null; + } + + fn redirectFor(self: *const MockServer, target: []const u8) ?[]const u8 { + for (self.redirects) |redirect| { + if (std.mem.eql(u8, redirect.path, target)) return redirect.location; + } + return null; + } + + fn nowNs(io: std.Io) i64 { + return @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); + } + + /// Whether this GET is over the rate window. Counts only the requests + /// it lets through, so a throttled one does not consume the budget. + fn overWindow(self: *MockServer, io: std.Io) bool { + const window = self.throttle.window orelse return false; + self.window_mutex.lockUncancelable(io); + defer self.window_mutex.unlock(io); + const now = nowNs(io); + if (now - self.window_started_ns > window.per_ms * std.time.ns_per_ms) { + self.window_started_ns = now; + self.window_gets = 0; + } + if (self.window_gets >= window.gets) return true; + self.window_gets += 1; + return false; + } + + fn respondThrottled(self: *MockServer, io: std.Io, request: *std.http.Server.Request) !void { + _ = self.first_throttle_ns.cmpxchgStrong(0, nowNs(io), .release, .monotonic); + var retry_after_buffer: [16]u8 = undefined; + try request.respond("slow down", .{ + .status = self.throttle.status, + .keep_alive = false, + .extra_headers = if (self.throttle.retry_after_s) |seconds| &.{.{ + .name = "Retry-After", + .value = try std.fmt.bufPrint(&retry_after_buffer, "{d}", .{seconds}), + }} else &.{}, + }); + } + + pub fn port(self: *const MockServer) u16 { + return self.tcp.socket.address.getPort(); + } + + fn run(self: *MockServer, io: std.Io) std.Io.Cancelable!void { + var connections: std.Io.Group = .init; + defer connections.cancel(io); + + while (true) { + const stream = self.tcp.accept(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + _ = self.first_error.set(err); + return; + }, + }; + connections.concurrent(io, onConnection, .{ self, stream, io }) catch |err| { + stream.close(io); + _ = self.first_error.set(err); + return; + }; + } + } + + fn onConnection( + self: *MockServer, + stream: std.Io.net.Stream, + io: std.Io, + ) std.Io.Cancelable!void { + defer stream.close(io); + self.handleConnection(stream, io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => _ = self.first_error.set(err), + }; + } + + fn handleConnection(self: *MockServer, stream: std.Io.net.Stream, io: std.Io) !void { + var read_buffer: [8 * 1024]u8 = undefined; + var reader = stream.reader(io, &read_buffer); + var write_buffer: [8 * 1024]u8 = undefined; + var writer = stream.writer(io, &write_buffer); + var server: std.http.Server = .init(&reader.interface, &writer.interface); + var request = try server.receiveHead(); + + if (self.redirectFor(request.head.target)) |location| { + try request.respond("", .{ + .status = .found, + .keep_alive = false, + .extra_headers = &.{.{ .name = "Location", .value = location }}, + }); + return; + } + + const body = self.bodyFor(request.head.target) orelse { + const refusal = self.refusalFor(request.head.target); + try request.respond("refused", .{ + .status = refusal orelse .not_found, + .keep_alive = false, + }); + if (refusal != null) return; + return error.UnexpectedTarget; + }; + + switch (request.head.method) { + .HEAD => { + const head_ordinal = self.head_requests.fetchAdd(1, .acq_rel) + 1; + if (head_ordinal <= self.throttle.first_heads) { + try self.respondThrottled(io, &request); + return; + } + try request.respond(body, .{ .keep_alive = false }); + }, + .GET => { + const range = try parseRequestRange(&request, body.len); + const get_ordinal = self.get_requests.fetchAdd(1, .acq_rel) + 1; + const active = self.active_gets.fetchAdd(1, .acq_rel) + 1; + _ = self.peak_gets.fetchMax(active, .acq_rel); + defer _ = self.active_gets.fetchSub(1, .release); + + if (self.barrier_gets != 0 and get_ordinal <= self.barrier_gets) { + if (get_ordinal == self.barrier_gets) self.get_barrier.set(io); + try self.get_barrier.wait(io); + } + + if (get_ordinal <= self.fail_first_gets) { + try request.respond("retry", .{ + .status = .internal_server_error, + .keep_alive = false, + }); + return; + } + + if (self.throttle.always or get_ordinal <= self.throttle.first_gets or self.overWindow(io)) { + try self.respondThrottled(io, &request); + return; + } + if (self.first_throttle_ns.load(.acquire) != 0) { + _ = self.first_get_after_throttle_ns.cmpxchgStrong(0, nowNs(io), .release, .monotonic); + } + + var content_range_buffer: [96]u8 = undefined; + const content_range = try std.fmt.bufPrint( + &content_range_buffer, + "bytes {d}-{d}/{d}", + .{ range.start, range.end, body.len }, + ); + try request.respond(body[range.start .. range.end + 1], .{ + .status = .partial_content, + .keep_alive = false, + .extra_headers = &.{ + .{ .name = "Content-Range", .value = content_range }, + }, + }); + }, + else => { + try request.respond("method not allowed", .{ + .status = .method_not_allowed, + .keep_alive = false, + }); + return error.UnexpectedMethod; + }, + } + } + + const Range = struct { + start: usize, + end: usize, + }; + + fn parseRequestRange(request: *const std.http.Server.Request, object_len: usize) !Range { + var value: ?[]const u8 = null; + var headers = request.iterateHeaders(); + while (headers.next()) |header| { + if (!std.ascii.eqlIgnoreCase(header.name, "Range")) continue; + if (value != null) return error.DuplicateRange; + value = header.value; + } + + const range_value = value orelse return error.MissingRange; + const prefix = "bytes="; + if (range_value.len <= prefix.len or + !std.ascii.eqlIgnoreCase(range_value[0..prefix.len], prefix)) + { + return error.InvalidRange; + } + const dash = std.mem.indexOfScalar(u8, range_value[prefix.len..], '-') orelse + return error.InvalidRange; + const absolute_dash = prefix.len + dash; + if (absolute_dash == prefix.len or absolute_dash + 1 == range_value.len) + return error.InvalidRange; + + const result: Range = .{ + .start = try std.fmt.parseInt(usize, range_value[prefix.len..absolute_dash], 10), + .end = try std.fmt.parseInt(usize, range_value[absolute_dash + 1 ..], 10), + }; + if (result.end < result.start or result.end >= object_len) return error.InvalidRange; + return result; + } +}; diff --git a/vfs/vfs.zig b/vfs/vfs.zig index c3211657..ef2fa1d5 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -21,6 +21,8 @@ pub const GCS = @import("gcs.zig").GCS; pub const HF = @import("hf.zig").HF; pub const HTTP = @import("http.zig").HTTP; pub const S3 = @import("s3.zig").S3; +/// A test-only HTTP origin for tests above this layer (mock_server.zig). +pub const MockServer = @import("mock_server.zig").MockServer; const base_module = @import("base.zig"); const direct_io = @import("direct_io.zig"); pub const Backend = base_module.Backend; From 5ac47400c61a9e0227ed6d6f551dbd0e2d57ea0e Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:00:06 +0100 Subject: [PATCH 033/103] vfs: add the governed request loop and the open/read error contract Each remote backend sends its requests with code of its own: HTTP retries nothing, and HF, S3 and GCS each carry a copy of the same retry loop that differs in which failures it retries and how it reports the rest. request.zig holds one loop for all of them. A backend owns a Governor, with its name, its retry knobs and its request statistics, and runs each request through perform, which retries a retryable failure after a full-jitter backoff, or after the delay the server named in Retry-After or HF's RateLimit header, up to max_delay, and gives up after max_retries. exchange owns the std.http plumbing of one attempt: it forces identity encoding, keeps its own redirect buffer, closes a connection after a partial send, and tells a canceled read from a failed one. A 408 is a timeout, a 429 a throttle, a 5xx a server failure, and a 503 either a throttle or a server failure as the backend says; any other status fails the request, logged once at err. Open, stat and access requests share one error contract: 404 and 410 are FileNotFound, 401 and 403 PermissionDenied, a cancel stays Canceled, and anything else is Unexpected, logged once by the governor. A failed read is Canceled or Unexpected. No backend uses the loop yet; its tests drive it, one of them against MockServer. Co-Authored-By: Claude Opus 5.5 --- vfs/BUILD.bazel | 1 + vfs/request.zig | 613 ++++++++++++++++++++++++++++++++++++++++++++++++ vfs/vfs.zig | 1 + 3 files changed, 615 insertions(+) create mode 100644 vfs/request.zig diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index d846af65..2b062e6e 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -10,6 +10,7 @@ zig_library( "http.zig", "mock_server.zig", "parallel_read.zig", + "request.zig", "s3.zig", ], main = "vfs.zig", diff --git a/vfs/request.zig b/vfs/request.zig new file mode 100644 index 00000000..3903691d --- /dev/null +++ b/vfs/request.zig @@ -0,0 +1,613 @@ +//! One governed HTTP request loop for every backend: send, classify, and +//! retry a retryable failure after a per-request full-jitter backoff, or +//! after the delay the server names. A backend's open, stat and read +//! failures map to one error contract (`OpenError`, +//! `Governor.readFailure`), each failure logged once. + +const std = @import("std"); + +const log = @import("base.zig").scopedLog(.@"zml/vfs/request"); + +/// The retry knobs of one remote backend: its `Options.retry`. +pub const Retry = struct { + /// Retries per request. + max_retries: usize = 5, + initial_delay: std.Io.Duration = .fromMilliseconds(500), + /// Longest backoff before a request's next attempt, a server-named delay + /// included. + max_delay: std.Io.Duration = .fromSeconds(30), +}; + +/// One per backend instance: what each of its requests needs and its read +/// statistics. +pub const Governor = struct { + pub const Stats = struct { + physical_requests: std.atomic.Value(u64) = .init(0), + physical_bytes: std.atomic.Value(u64) = .init(0), + retries: std.atomic.Value(u64) = .init(0), + transient_retries: std.atomic.Value(u64) = .init(0), + timeouts: std.atomic.Value(u64) = .init(0), + server_failures: std.atomic.Value(u64) = .init(0), + throttles: std.atomic.Value(u64) = .init(0), + retry_delay_ns: std.atomic.Value(u64) = .init(0), + + fn recordAttempt(self: *Stats) void { + _ = self.physical_requests.fetchAdd(1, .monotonic); + } + + pub fn recordSuccess(self: *Stats, request_size: usize) void { + _ = self.physical_bytes.fetchAdd(@intCast(request_size), .monotonic); + } + + fn recordFailure(self: *Stats, failure: ReadFailure) void { + switch (failure) { + .transient => _ = self.transient_retries.fetchAdd(1, .monotonic), + .timeout => _ = self.timeouts.fetchAdd(1, .monotonic), + .server_failure => _ = self.server_failures.fetchAdd(1, .monotonic), + .throttle => _ = self.throttles.fetchAdd(1, .monotonic), + } + } + + fn recordRetry(self: *Stats) void { + _ = self.retries.fetchAdd(1, .monotonic); + } + + fn recordRetryDelay(self: *Stats, delay: std.Io.Duration) void { + const delay_ns: u64 = @intCast(@max(delay.nanoseconds, 0)); + _ = self.retry_delay_ns.fetchAdd(delay_ns, .monotonic); + } + }; + + io: std.Io, + client: *std.http.Client, + /// The backend's name, for log lines. + name: []const u8, + /// What a 503 means for this backend: rate limiting on the object stores + /// (AWS `SlowDown`, GCS), a server failure elsewhere. + unavailable: ReadFailure, + retry: Retry, + stats: Stats = .{}, + + pub fn init(io: std.Io, client: *std.http.Client, name: []const u8, unavailable: ReadFailure, retry: Retry) Governor { + std.debug.assert(retry.initial_delay.nanoseconds >= 0); + std.debug.assert(retry.max_delay.nanoseconds >= retry.initial_delay.nanoseconds); + return .{ .io = io, .client = client, .name = name, .unavailable = unavailable, .retry = retry }; + } + + /// Logs the read totals at debug, once, from the backend's `deinit`; + /// nothing when the backend sent no request. Retries and failures are + /// logged at warn as they happen. + pub fn logTotals(self: *const Governor) void { + const stats = &self.stats; + const requests = stats.physical_requests.load(.acquire); + if (requests == 0) return; + // Under the VFS's scope, `zml/vfs`. + std.log.scoped(.@"zml/vfs").debug("{s} read totals: requests={d}, bytes={Bi:.2}, retries={d}, transient_retries={d}, timeouts={d}, server_failures={d}, throttles={d}, retry_delay_ms={d:.3}", .{ + self.name, + requests, + stats.physical_bytes.load(.acquire), + stats.retries.load(.acquire), + stats.transient_retries.load(.acquire), + stats.timeouts.load(.acquire), + stats.server_failures.load(.acquire), + stats.throttles.load(.acquire), + @as(f64, @floatFromInt(stats.retry_delay_ns.load(.acquire))) / std.time.ns_per_ms, + }); + } + + /// A failed open, stat or access of `path` as `OpenError` reports it. + pub fn openFailure(self: *const Governor, path: []const u8, err: anyerror) OpenError { + return switch (err) { + error.FileNotFound => error.FileNotFound, + error.PermissionDenied => error.PermissionDenied, + error.Canceled => error.Canceled, + // Logged by `exchange`. + error.RequestFailed => error.Unexpected, + error.OutOfMemory => error.SystemResources, + else => { + log.err("{s}: looking up {s} failed: {t}", .{ self.name, path, err }); + return error.Unexpected; + }, + }; + } + + /// A failed read of `target` at `offset` as a backend's read reports it: + /// a canceled task stays `Canceled`, since every wait in the governed + /// loop is a cancellation point; any other failure is logged once at + /// err, by `exchange` or here, and is `Unexpected`. + pub fn readFailure(self: *const Governor, target: []const u8, offset: u64, err: anyerror) error{ Canceled, Unexpected } { + return switch (err) { + error.Canceled => error.Canceled, + // Logged by `exchange`. + error.RequestFailed => error.Unexpected, + else => { + log.err("{s}: reading {s} at {d} failed: {t}", .{ self.name, target, offset, err }); + return error.Unexpected; + }, + }; + } +}; + +/// One attempt as a backend's request hook sees it. +pub const Attempt = struct { + /// The backend and the request as `perform` was given them, which + /// `exchange` sends and logs with. + governor: *Governor, + spec: RequestSpec, + /// The status the previous attempt returned, for a hook that reacts to + /// one (an expired signed URL answering 403). + previous_status: ?std.http.Status = null, +}; + +/// Identity of one request, for the logs and for the classification. +pub const RequestSpec = struct { + /// Object identity for log lines. + target: []const u8, + /// Statuses this request retries although they are not retryable in + /// general, because its hook can do something about them: a signed + /// download URL that expired answers 401 or 403, and the hook + /// re-resolves it. Charged to the retry budget. + retry_once: []const std.http.Status = &.{}, +}; + +pub const ReadFailure = enum { + transient, + timeout, + server_failure, + throttle, +}; + +pub const Failure = struct { + failure: ReadFailure, + /// The delay the server named, if any. + delay: ?std.Io.Duration = null, + status: ?std.http.Status = null, +}; + +/// What one attempt produced: a value, or a reason to try again. +pub fn Outcome(comptime T: type) type { + return union(enum) { + done: T, + retry: Failure, + }; +} + +/// The one request loop. `attemptFn` performs a single HTTP exchange (see +/// `exchange`); this drives the classification and the retry budget around +/// it. Every retryable failure, a throttle included, sleeps the per-request +/// backoff, or the delay the server named up to `max_delay`, and charges one +/// of `max_retries`. A cancel of the caller propagates. +pub fn perform( + comptime T: type, + governor: *Governor, + spec: RequestSpec, + context: anytype, + comptime attemptFn: fn (@TypeOf(context), Attempt) anyerror!Outcome(T), +) anyerror!T { + var charged: usize = 0; + var previous_status: ?std.http.Status = null; + while (true) { + governor.stats.recordAttempt(); + switch (try attemptFn(context, .{ .governor = governor, .spec = spec, .previous_status = previous_status })) { + .done => |value| return value, + .retry => |failure| { + governor.stats.recordFailure(failure.failure); + previous_status = failure.status; + if (charged >= governor.retry.max_retries) return error.RetriesExhausted; + charged += 1; + governor.stats.recordRetry(); + const retry = governor.retry; + const delay = if (failure.delay) |named| + clampDelay(named, .fromNanoseconds(0), retry.max_delay) + else + fullJitterDelay(governor.io, retry.initial_delay, retry.max_delay, charged); + governor.stats.recordRetryDelay(delay); + try governor.io.sleep(delay, .awake); + }, + } + } +} + +fn clampDelay(value: std.Io.Duration, low: std.Io.Duration, high: std.Io.Duration) std.Io.Duration { + return .fromNanoseconds(std.math.clamp(value.nanoseconds, low.nanoseconds, high.nanoseconds)); +} + +/// The hops `Redirects.follow` takes: std's default. +pub const max_redirects = 3; + +/// What happens to a redirect: the client chases it (`max_redirects` hops), +/// `consume` receives it to walk the chain itself, or it fails the request. +pub const Redirects = enum { + follow, + surface, + forbid, + + fn behavior(self: Redirects) std.http.Client.Request.RedirectBehavior { + return switch (self) { + .follow => .init(max_redirects), + .surface => .unhandled, + .forbid => .not_allowed, + }; + } +}; + +pub const ExchangeOptions = struct { + method: std.http.Method = .GET, + authorization: std.http.Client.Request.Headers.Value = .default, + content_type: std.http.Client.Request.Headers.Value = .default, + extra_headers: []const std.http.Header = &.{}, + /// Sent as the whole request body; the method must allow one. The + /// writer uses it as its own buffer, so it must be mutable. + payload: ?[]u8 = null, + /// An open, stat or access: an absent or denied object fails the + /// request with its `OpenError` before `consume`, unlogged. + open: bool = false, + redirects: Redirects = .forbid, +}; + +/// One HTTP exchange of `attempt`'s request: send, receive the head, +/// classify it, and hand a 2xx (or a redirect `Redirects.surface` hands +/// back) to `consume` while the request is alive. Retryable connection, +/// send and receive failures and every classified status become `.retry`; +/// an unclassified status or any other connect or head failure fails with +/// `error.RequestFailed`, logged at err, except that an open request +/// (`ExchangeOptions.open`) fails an absent or denied object with +/// `FileNotFound` or `PermissionDenied` first, unlogged. +pub fn exchange( + comptime T: type, + attempt: Attempt, + uri: std.Uri, + options: ExchangeOptions, + context: anytype, + comptime consume: fn (@TypeOf(context), *std.http.Client.Response) anyerror!T, +) anyerror!Outcome(T) { + const spec = attempt.spec; + // A followed redirect keeps each hop's Location and merged path here, + // 8 KiB of each; `req.uri` points into it. + var redirect_buffer: [2 * max_redirects * 8 * 1024]u8 = undefined; + var req = attempt.governor.client.request(options.method, uri, .{ + .redirect_behavior = options.redirects.behavior(), + .headers = .{ + // Range offsets and Content-Length count the stored bytes. + .accept_encoding = .{ .override = "identity" }, + .authorization = options.authorization, + .content_type = options.content_type, + }, + .extra_headers = options.extra_headers, + }) catch |err| switch (err) { + error.Timeout => return retryable(T, attempt, "connect", err, .timeout), + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.NameServerFailure, + => return retryable(T, attempt, "connect", err, .transient), + // The caller's cancel. + error.Canceled => return err, + error.OutOfMemory => return err, + else => return fatal(attempt, "connect", err), + }; + defer req.deinit(); + + // A request that was not sent whole leaves its connection unusable; + // `deinit` would otherwise pool it, its reader still `ready`. + if (options.payload) |payload| { + req.sendBodyComplete(payload) catch |err| switch (err) { + error.WriteFailed => { + req.connection.?.closing = true; + if (canceled(&req)) return error.Canceled; + return retryable(T, attempt, "send body", err, .transient); + }, + }; + } else { + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + req.connection.?.closing = true; + if (canceled(&req)) return error.Canceled; + return retryable(T, attempt, "send headers", err, .transient); + }, + }; + } + + var res = req.receiveHead(if (options.redirects == .follow) &redirect_buffer else &.{}) catch |err| { + // A failed read already closed it; a redirect re-sends on a new + // connection that a failure leaves unusable too (or null). + if (req.connection) |connection| connection.closing = true; + if (canceled(&req)) return error.Canceled; + switch (err) { + error.Timeout => return retryable(T, attempt, "receive headers", err, .timeout), + error.HttpConnectionClosing, + error.HttpRequestTruncated, + error.ReadFailed, + error.WriteFailed, + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.NameServerFailure, + => return retryable(T, attempt, "receive headers", err, .transient), + // The caller's cancel. + error.Canceled => return err, + error.OutOfMemory => return err, + else => return fatal(attempt, "receive headers", err), + } + }; + + if (options.open) { + if (openStatus(res.head.status)) |err| return err; + } + const class = res.head.status.class(); + if (class != .success and !(class == .redirect and options.redirects == .surface)) { + for (spec.retry_once) |status| { + if (status != res.head.status) continue; + log.warn("{s}: {s} answered {d}; retrying once with a fresh request", .{ + attempt.governor.name, + spec.target, + @backingInt(res.head.status), + }); + return .{ .retry = .{ .failure = .transient, .status = res.head.status } }; + } + const failure = classifyStatus(res.head.status, attempt.governor.unavailable) orelse { + log.err("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); + return error.RequestFailed; + }; + log.warn("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); + return .{ .retry = .{ + .failure = failure, + .delay = serverRetryDelay(res.head), + .status = res.head.status, + } }; + } + + return .{ .done = consume(context, &res) catch |err| switch (err) { + error.EndOfStream, error.ReadFailed => { + if (canceled(&req)) return error.Canceled; + return retryable(T, attempt, "read body", err, .transient); + }, + else => return err, + } }; +} + +/// Whether the exchange's read or write failed because it was canceled: +/// std's stream keeps the cancel and reports a failed read or write. +fn canceled(req: *const std.http.Client.Request) bool { + const connection = req.connection orelse return false; + if (connection.stream_reader.err) |err| return err == error.Canceled; + if (connection.stream_writer.err) |err| return err == error.Canceled; + return false; +} + +fn retryable( + comptime T: type, + attempt: Attempt, + stage: []const u8, + err: anyerror, + failure: ReadFailure, +) Outcome(T) { + log.warn("{s}: {s} for {s} failed: {}", .{ attempt.governor.name, stage, attempt.spec.target, err }); + return .{ .retry = .{ .failure = failure } }; +} + +fn fatal(attempt: Attempt, stage: []const u8, err: anyerror) error{RequestFailed} { + log.err("{s}: {s} for {s} failed: {}", .{ attempt.governor.name, stage, attempt.spec.target, err }); + return error.RequestFailed; +} + +/// Retry classification of a non-2xx status; null when the status is not +/// retried. `unavailable` is what a 503 means for the backend: rate limiting +/// on the object stores (AWS `SlowDown`, GCS), a server failure elsewhere. +fn classifyStatus(status: std.http.Status, unavailable: ReadFailure) ?ReadFailure { + return switch (status) { + .request_timeout => .timeout, + .too_many_requests => .throttle, + .service_unavailable => unavailable, + else => if (status.class() == .server_error) .server_failure else null, + }; +} + +/// What an open, stat or access of a remote path fails with: `FileNotFound` +/// for a missing object (404, 410), `PermissionDenied` for a denied one +/// (401, 403), `Canceled` for a canceled task and `SystemResources` when a +/// request cannot allocate. `openStatus` maps the statuses; anything else is +/// `Unexpected`, logged once, by `exchange` or `Governor.openFailure`. +pub const OpenError = error{ FileNotFound, PermissionDenied, Canceled, SystemResources, Unexpected }; + +/// The error a status means for the path an open request +/// (`ExchangeOptions.open`) asked about; null when the status is not one. +fn openStatus(status: std.http.Status) ?error{ FileNotFound, PermissionDenied } { + return switch (status) { + .not_found, .gone => error.FileNotFound, + .unauthorized, .forbidden => error.PermissionDenied, + else => null, + }; +} + +/// The retry delay the server names, if any: `Retry-After` delta-seconds +/// (the HTTP-date form is not parsed and falls back to the jittered delay) +/// or the `t=` reset of the `RateLimit` header Hugging Face sends. +fn serverRetryDelay(head: std.http.Client.Response.Head) ?std.Io.Duration { + var it = head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Retry-After")) { + return delaySeconds(header.value) orelse continue; + } + if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { + var parts = std.mem.splitScalar(u8, header.value, ';'); + while (parts.next()) |part| { + const trimmed = std.mem.trim(u8, part, " \t"); + if (std.mem.startsWith(u8, trimmed, "t=")) return delaySeconds(trimmed[2..]) orelse continue; + } + } + } + return null; +} + +fn delaySeconds(value: []const u8) ?std.Io.Duration { + const seconds = std.fmt.parseInt(u32, std.mem.trim(u8, value, " \t"), 10) catch return null; + return .fromSeconds(seconds); +} + +fn fullJitterDelay( + io: std.Io, + initial: std.Io.Duration, + maximum: std.Io.Duration, + attempt: usize, +) std.Io.Duration { + const max_delay_ns: i96 = @min( + maximum.toNanoseconds(), + initial.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), + ); + if (max_delay_ns <= 0) return .fromNanoseconds(0); + + var seed: u64 = undefined; + io.random(@ptrCast(&seed)); + var prng: std.Random.DefaultPrng = .init(seed); + return .fromNanoseconds(prng.random().intRangeAtMost(i96, 0, max_delay_ns)); +} + +const test_spec: RequestSpec = .{ .target = "object" }; + +fn testGovernor(initial_ms: i64, max_delay_ms: i64) Governor { + // No attempt in these tests reaches the network: no client. + return .init(std.testing.io, undefined, "test", .throttle, .{ + .max_retries = 3, + .initial_delay = .fromMilliseconds(initial_ms), + .max_delay = .fromMilliseconds(max_delay_ms), + }); +} + +test "a timeout and a server failure each charge one retry" { + var governor = testGovernor(0, 0); + + const Attempts = struct { + seen: usize = 0, + fn run(self: *@This(), _: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + return switch (self.seen) { + 1 => .{ .retry = .{ .failure = .timeout } }, + 2 => .{ .retry = .{ .failure = .server_failure } }, + else => .{ .done = self.seen }, + }; + } + }; + var attempts: Attempts = .{}; + try std.testing.expectEqual(@as(usize, 3), try perform(usize, &governor, test_spec, &attempts, Attempts.run)); + try std.testing.expectEqual(@as(u64, 2), governor.stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 3), governor.stats.physical_requests.load(.acquire)); +} + +test "the retry budget bounds retryable failures" { + var governor = testGovernor(0, 0); + + const Attempts = struct { + seen: usize = 0, + fn run(self: *@This(), _: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + return .{ .retry = .{ .failure = .transient } }; + } + }; + var attempts: Attempts = .{}; + try std.testing.expectError(error.RetriesExhausted, perform(usize, &governor, test_spec, &attempts, Attempts.run)); + // The first attempt plus `max_retries` of them. + try std.testing.expectEqual(@as(usize, 4), attempts.seen); +} + +test "retry status classification is typed and 503 depends on the backend" { + try std.testing.expectEqual(ReadFailure.timeout, classifyStatus(.request_timeout, .server_failure).?); + try std.testing.expectEqual(ReadFailure.throttle, classifyStatus(.too_many_requests, .server_failure).?); + try std.testing.expectEqual(ReadFailure.server_failure, classifyStatus(.bad_gateway, .throttle).?); + try std.testing.expectEqual(ReadFailure.server_failure, classifyStatus(.service_unavailable, .server_failure).?); + try std.testing.expectEqual(ReadFailure.throttle, classifyStatus(.service_unavailable, .throttle).?); + try std.testing.expect(classifyStatus(.not_found, .throttle) == null); +} + +test "absent and denied statuses map to one error each" { + try std.testing.expect(openStatus(.not_found).? == error.FileNotFound); + try std.testing.expect(openStatus(.gone).? == error.FileNotFound); + try std.testing.expect(openStatus(.unauthorized).? == error.PermissionDenied); + try std.testing.expect(openStatus(.forbidden).? == error.PermissionDenied); + for ([_]std.http.Status{ .ok, .partial_content, .bad_request, .too_many_requests, .internal_server_error, .service_unavailable }) |status| { + try std.testing.expect(openStatus(status) == null); + } + + const governor = testGovernor(10, 10); + try std.testing.expect(governor.openFailure("object", error.FileNotFound) == error.FileNotFound); + try std.testing.expect(governor.openFailure("object", error.PermissionDenied) == error.PermissionDenied); + try std.testing.expect(governor.openFailure("object", error.Canceled) == error.Canceled); + try std.testing.expect(governor.openFailure("object", error.OutOfMemory) == error.SystemResources); + try std.testing.expect(governor.openFailure("object", error.RequestFailed) == error.Unexpected); + try std.testing.expect(governor.openFailure("object", error.InvalidContentRange) == error.Unexpected); +} + +test "a failed read is Canceled or Unexpected" { + const governor = testGovernor(10, 10); + try std.testing.expect(governor.readFailure("object", 0, error.Canceled) == error.Canceled); + try std.testing.expect(governor.readFailure("object", 0, error.RequestFailed) == error.Unexpected); + try std.testing.expect(governor.readFailure("object", 0, error.InvalidContentRange) == error.Unexpected); +} + +test "server retry delay comes from Retry-After seconds or a RateLimit reset" { + const Head = std.http.Client.Response.Head; + const retry_after = try Head.parse("HTTP/1.1 503 Service Unavailable\r\nretry-after: 3\r\n\r\n"); + try std.testing.expectEqual(std.Io.Duration.fromSeconds(3), serverRetryDelay(retry_after).?); + + const rate_limit = try Head.parse("HTTP/1.1 429 Too Many Requests\r\nRateLimit: \"default\"; r=0; t=7\r\n\r\n"); + try std.testing.expectEqual(std.Io.Duration.fromSeconds(7), serverRetryDelay(rate_limit).?); + + const http_date = try Head.parse("HTTP/1.1 503 Service Unavailable\r\nRetry-After: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"); + try std.testing.expect(serverRetryDelay(http_date) == null); + const none = try Head.parse("HTTP/1.1 500 Internal Server Error\r\n\r\n"); + try std.testing.expect(serverRetryDelay(none) == null); +} + +test "a server-named retry delay sleeps at most max_delay" { + var governor = testGovernor(1, 20); + const Failing = struct { + seen: usize = 0, + fn attempt(self: *@This(), _: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + // A failing server that asks for an hour. + if (self.seen == 1) return .{ .retry = .{ .failure = .server_failure, .delay = .fromSeconds(3600) } }; + return .{ .done = self.seen }; + } + }; + var failing: Failing = .{}; + try std.testing.expectEqual(@as(usize, 2), try perform(usize, &governor, test_spec, &failing, Failing.attempt)); + try std.testing.expectEqual(@as(u64, 20 * std.time.ns_per_ms), governor.stats.retry_delay_ns.load(.acquire)); +} + +test "an exchange canceled while it waits for the head returns Canceled, not a retry" { + const MockServer = @import("mock_server.zig").MockServer; + const io = std.testing.io; + + // The GET waits unanswered for a second one that never comes. + var server = try MockServer.init(io, "object", .{ .barrier_gets = 2 }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = std.testing.allocator, .io = io }; + defer client.deinit(); + var governor: Governor = .init(io, &client, "test", .throttle, .{}); + var url_buffer: [64]u8 = undefined; + const uri: std.Uri = try .parse(try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()})); + + const Exchange = struct { + fn run(governor_: *Governor, uri_: std.Uri, result: *anyerror!Outcome(void)) void { + result.* = exchange(void, .{ .governor = governor_, .spec = test_spec }, uri_, .{ + .extra_headers = &.{.{ .name = "Range", .value = "bytes=0-5" }}, + }, {}, consume); + } + + fn consume(_: void, _: *std.http.Client.Response) anyerror!void {} + }; + var result: anyerror!Outcome(void) = .{ .done = {} }; + var group: std.Io.Group = .init; + try group.concurrent(io, Exchange.run, .{ &governor, uri, &result }); + try io.sleep(.fromMilliseconds(50), .awake); + group.cancel(io); + try std.testing.expectError(error.Canceled, result); + + try server.stop(io); +} diff --git a/vfs/vfs.zig b/vfs/vfs.zig index ef2fa1d5..120a962f 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -36,6 +36,7 @@ pub const direct_io_alignment: usize = if (direct_io.supported) direct_io.alignm test { _ = @import("gcs.zig"); + _ = @import("request.zig"); } const log = std.log.scoped(.@"zml/vfs"); From a82cf65654c9c2e6041fa9005b93e46803837ef4 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:01:01 +0100 Subject: [PATCH 034/103] vfs: add range_read, one Range GET per positional read performRangeRead turns one positional read into one GET with a Range header for what the caller's buffers hold of the object, run through the governed loop, and scatters the response into every buffer. Nothing is sent at the end of the object. A 206 must carry a Content-Range that covers the request, parsed strictly; a 200 that ignored the Range is positioned by discarding up to the offset. The backend supplies the request through a prepare hook, called once per attempt, since an S3 signature covers the Range header and a timestamp and a token or a signed URL may expire between attempts; static serves a plain GET. It replaces parallel_read's chunk reader and HTTP's own Content-Range parsing once the backends move to it. Co-Authored-By: Claude Opus 5.5 --- vfs/BUILD.bazel | 2 + vfs/http_acceptance_test.zig | 79 ++++++++++++ vfs/range_read.zig | 240 +++++++++++++++++++++++++++++++++++ vfs/vfs.zig | 2 +- 4 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 vfs/http_acceptance_test.zig create mode 100644 vfs/range_read.zig diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index 2b062e6e..366c6488 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -8,8 +8,10 @@ zig_library( "gcs.zig", "hf.zig", "http.zig", + "http_acceptance_test.zig", "mock_server.zig", "parallel_read.zig", + "range_read.zig", "request.zig", "s3.zig", ], diff --git a/vfs/http_acceptance_test.zig b/vfs/http_acceptance_test.zig new file mode 100644 index 00000000..5f68c410 --- /dev/null +++ b/vfs/http_acceptance_test.zig @@ -0,0 +1,79 @@ +const std = @import("std"); + +const MockServer = @import("mock_server.zig").MockServer; +const range_read = @import("range_read.zig"); +const vfs_request = @import("request.zig"); + +/// A retry configuration that never sleeps, for the tests that count +/// attempts rather than time. +fn instantRetries(max_retries: usize) vfs_request.Retry { + return .{ + .max_retries = max_retries, + .initial_delay = .fromNanoseconds(0), + .max_delay = .fromNanoseconds(0), + }; +} + +test "the shared range loop prepares the request once per attempt" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const request_size = 3 * 1024; + + var object: [request_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 7); + var output: [request_size]u8 = undefined; + + var server = try MockServer.init(io, &object, .{ .fail_first_gets = 2 }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + // An S3-style request: as a SigV4 signature must be, it is prepared + // again by the hook for every attempt. + const Signed = struct { + uri: std.Uri, + prepared: usize = 0, + + fn prepare(self: *@This(), _: vfs_request.Attempt, range: std.http.Header) anyerror!range_read.PreparedRequest { + try std.testing.expectEqualStrings("Range", range.name); + self.prepared += 1; + return .{ .uri = self.uri }; + } + }; + var url_buffer: [160]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + var signed: Signed = .{ .uri = try .parse(url) }; + const spec: range_read.RangeSpec = .{ .request = .{ .target = url } }; + const buffers = [_][]u8{ output[0..1024], output[1024..] }; + + // No retry budget: one attempt, one hook call, the failure is counted. + var no_retry: vfs_request.Governor = .init(io, &client, "test", .throttle, instantRetries(0)); + try std.testing.expectError( + error.RetriesExhausted, + range_read.performRangeRead(&no_retry, spec, &signed, Signed.prepare, &buffers, 0, object.len), + ); + try std.testing.expectEqual(@as(usize, 1), signed.prepared); + try std.testing.expectEqual(@as(u64, 1), no_retry.stats.server_failures.load(.acquire)); + + signed.prepared = 0; + var one_retry: vfs_request.Governor = .init(io, &client, "test", .throttle, instantRetries(1)); + try std.testing.expectEqual( + object.len, + try range_read.performRangeRead(&one_retry, spec, &signed, Signed.prepare, &buffers, 0, object.len), + ); + try std.testing.expectEqual(@as(usize, 2), signed.prepared); + try std.testing.expectEqualSlices(u8, &object, &output); + + try server.stop(io); + + // One GET per hook call. + try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); + const stats = &one_retry.stats; + try std.testing.expectEqual(@as(u64, 2), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); +} diff --git a/vfs/range_read.zig b/vfs/range_read.zig new file mode 100644 index 00000000..e2ce3ff0 --- /dev/null +++ b/vfs/range_read.zig @@ -0,0 +1,240 @@ +//! The Range specifics of a governed request: the `Range` header, the +//! `Content-Range` validation and the scatter into the caller's buffers. +//! Everything else (the hold, the retries, the classification) is in +//! `request.zig`. + +const std = @import("std"); + +const request = @import("request.zig"); + +const ContentRange = struct { + start: u64, + end: u64, + total: u64, +}; + +/// The backend's part of one GET: the URI, the authorization value and up to +/// `max_extra_headers` headers. The loop appends `Range`. Every slice must +/// stay valid until the attempt completes, so hooks write into storage that +/// outlives the call (the backend's per-read context). +pub const PreparedRequest = struct { + uri: std.Uri, + authorization: std.http.Client.Request.Headers.Value = .default, + extra_headers: []const std.http.Header = &.{}, +}; + +pub const max_extra_headers = 4; + +/// One range read: what identifies it, and whether it follows a redirect. +pub const RangeSpec = struct { + request: request.RequestSpec, + /// The object stores and a resolved download URL never redirect a + /// read; a plain HTTP server may. + redirects: request.Redirects = .forbid, +}; + +/// The `prepare` of a request that is the same on every attempt: a plain +/// GET of `uri`. +pub fn static(uri: *const std.Uri, _: request.Attempt, _: std.http.Header) anyerror!PreparedRequest { + return .{ .uri = uri.* }; +} + +/// One `GET` with `Range: bytes=offset-(offset+size-1)` per attempt through +/// the governed loop, where `size` is what `data` holds of the `file_size` +/// bytes past `offset`; nothing is sent when it is 0. `prepare` is called +/// once per attempt: S3's SigV4 signature covers the `Range` header and a +/// timestamp, a bearer token may have expired, and a signed download URL +/// may need re-resolving after a 403. +/// The caller holds its read slot through every retry and sleep; +/// backend-local parallel readers would multiply the caller's width and +/// defeat its memory bounds. +pub fn performRangeRead( + governor: *request.Governor, + spec: RangeSpec, + context: anytype, + comptime prepare: fn (@TypeOf(context), request.Attempt, std.http.Header) anyerror!PreparedRequest, + data: []const []u8, + offset: u64, + file_size: u64, +) !usize { + const size = readSize(file_size, offset, data); + if (size == 0) return 0; + + const RangeRead = struct { + context: @TypeOf(context), + redirects: request.Redirects, + range: std.http.Header, + data: []const []u8, + offset: u64, + size: usize, + + fn attempt(self: *@This(), current: request.Attempt) anyerror!request.Outcome(void) { + const prepared = try prepare(self.context, current, self.range); + std.debug.assert(prepared.extra_headers.len <= max_extra_headers); + var headers: [max_extra_headers + 1]std.http.Header = undefined; + @memcpy(headers[0..prepared.extra_headers.len], prepared.extra_headers); + headers[prepared.extra_headers.len] = self.range; + return request.exchange(void, current, prepared.uri, .{ + .authorization = prepared.authorization, + .extra_headers = headers[0 .. prepared.extra_headers.len + 1], + .redirects = self.redirects, + }, self, consume); + } + + fn consume(self: *@This(), res: *std.http.Client.Response) anyerror!void { + // The head bytes are released when the body reader is taken. + const content_range = contentRange(res.head); + return readResponse(res.reader(&.{}), res.head.status, content_range, self.offset, self.data, self.size); + } + }; + + var range_buffer: [64]u8 = undefined; + var state: RangeRead = .{ + .context = context, + .redirects = spec.redirects, + .range = .{ + .name = "Range", + .value = std.fmt.bufPrint( + &range_buffer, + "bytes={d}-{d}", + .{ offset, offset + @as(u64, @intCast(size - 1)) }, + ) catch unreachable, + }, + .data = data, + .offset = offset, + .size = size, + }; + try request.perform(void, governor, spec.request, &state, RangeRead.attempt); + governor.stats.recordSuccess(size); + return size; +} + +fn contentRange(head: std.http.Client.Response.Head) ?ContentRange { + var it = head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) return parseContentRange(header.value); + } + return null; +} + +fn parseContentRange(value: []const u8) ?ContentRange { + const prefix = "bytes "; + if (value.len < prefix.len or !std.ascii.eqlIgnoreCase(value[0..prefix.len], prefix)) return null; + const range_and_total = value[prefix.len..]; + const dash = std.mem.indexOfScalar(u8, range_and_total, '-') orelse return null; + const slash = dash + 1 + (std.mem.indexOfScalar(u8, range_and_total[dash + 1 ..], '/') orelse return null); + if (dash == 0 or slash == dash + 1 or slash + 1 == range_and_total.len) return null; + if (std.mem.indexOfScalar(u8, range_and_total[slash + 1 ..], '/') != null) return null; + + const result: ContentRange = .{ + .start = std.fmt.parseInt(u64, range_and_total[0..dash], 10) catch return null, + .end = std.fmt.parseInt(u64, range_and_total[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, range_and_total[slash + 1 ..], 10) catch return null, + }; + if (result.end < result.start or result.end >= result.total) return null; + return result; +} + +fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { + if (offset >= file_size) return 0; + + var requested: usize = 0; + for (data) |buf| requested +|= buf.len; + return @intCast(@min(file_size - offset, requested)); +} + +/// Validates the response against the requested range, discards the prefix +/// of a `200` that ignored `Range`, then scatters `read_size` bytes. +fn readResponse( + reader: *std.Io.Reader, + status: std.http.Status, + content_range: ?ContentRange, + offset: u64, + data: []const []u8, + read_size: usize, +) !void { + std.debug.assert(read_size != 0); + + const response_start = switch (status) { + .partial_content => blk: { + const cr = content_range orelse return error.InvalidContentRange; + const requested_end = std.math.add(u64, offset, read_size - 1) catch return error.InvalidContentRange; + if (cr.start > offset or cr.end < requested_end) return error.InvalidContentRange; + break :blk cr.start; + }, + .ok => 0, + else => return error.UnexpectedStatus, + }; + + try reader.discardAll64(offset - response_start); + try readScatter(reader, data, read_size); +} + +fn readScatter(reader: *std.Io.Reader, data: []const []u8, len: usize) !void { + var remaining = len; + for (data) |buf| { + const destination = buf[0..@min(remaining, buf.len)]; + remaining -= destination.len; + try reader.readSliceAll(destination); + } + if (remaining != 0) return error.UnexpectedEndOfOutput; +} + +test "Content-Range parsing is strict" { + try std.testing.expectEqual( + ContentRange{ .start = 2, .end = 9, .total = 10 }, + parseContentRange("bytes 2-9/10").?, + ); + try std.testing.expectEqual( + ContentRange{ .start = 2, .end = 9, .total = 10 }, + parseContentRange("ByTeS 2-9/10").?, + ); + try std.testing.expect(parseContentRange("items 2-9/10") == null); + try std.testing.expect(parseContentRange("bytes 2-9/*") == null); + try std.testing.expect(parseContentRange("bytes 9-2/10") == null); + try std.testing.expect(parseContentRange("bytes 2-10/10") == null); + try std.testing.expect(parseContentRange("bytes 2-9/10 trailing") == null); +} + +test "range responses fill scatter buffers" { + var reader: std.Io.Reader = .fixed("23456789"); + var first: [2]u8 = undefined; + var second: [3]u8 = undefined; + try readResponse( + &reader, + .partial_content, + .{ .start = 2, .end = 9, .total = 10 }, + 3, + &.{ &first, &second }, + 5, + ); + try std.testing.expectEqualStrings("34", &first); + try std.testing.expectEqualStrings("567", &second); +} + +test "200 responses that ignore Range are positioned and scattered" { + var reader: std.Io.Reader = .fixed("0123456789"); + var first: [1]u8 = undefined; + var second: [4]u8 = undefined; + try readResponse(&reader, .ok, null, 3, &.{ &first, &second }, 5); + try std.testing.expectEqualStrings("3", &first); + try std.testing.expectEqualStrings("4567", &second); +} + +test "partial responses require a covering Content-Range" { + var reader: std.Io.Reader = .fixed("3456"); + var output: [4]u8 = undefined; + try std.testing.expectError(error.InvalidContentRange, readResponse(&reader, .partial_content, null, 3, &.{&output}, 4)); + + reader = .fixed("4567"); + try std.testing.expectError( + error.InvalidContentRange, + readResponse(&reader, .partial_content, .{ .start = 4, .end = 7, .total = 10 }, 3, &.{&output}, 4), + ); + + reader = .fixed("345"); + try std.testing.expectError( + error.InvalidContentRange, + readResponse(&reader, .partial_content, .{ .start = 3, .end = 5, .total = 10 }, 3, &.{&output}, 4), + ); +} diff --git a/vfs/vfs.zig b/vfs/vfs.zig index 120a962f..6d39dc4c 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -36,7 +36,7 @@ pub const direct_io_alignment: usize = if (direct_io.supported) direct_io.alignm test { _ = @import("gcs.zig"); - _ = @import("request.zig"); + _ = @import("http_acceptance_test.zig"); } const log = std.log.scoped(.@"zml/vfs"); From 3b46505e9e726c53b5801f6ebc7bc628f2e28db9 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:01:32 +0100 Subject: [PATCH 035/103] vfs: add HeadChain, a governed HEAD redirect walker std.http.Client does not follow a redirect for a HEAD, so a backend that sizes an object with a HEAD walks the chain itself. HTTP's walk stopped at a relative Location (Hugging Face answers a resolve URL with one), and HF carries a walk of its own. HeadChain follows absolute and relative Locations through three rotating 8 KiB buffers, up to a hop limit the caller sets, and returns the URL that answered a 2xx with its Content-Length. Each hop is one governed HEAD with open semantics, so a missing or denied object fails the walk with FileNotFound or PermissionDenied, and a callback chooses the credentials sent to each URL. Co-Authored-By: Claude Opus 5.5 --- vfs/request.zig | 84 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/vfs/request.zig b/vfs/request.zig index 3903691d..79351cce 100644 --- a/vfs/request.zig +++ b/vfs/request.zig @@ -216,7 +216,8 @@ fn clampDelay(value: std.Io.Duration, low: std.Io.Duration, high: std.Io.Duratio pub const max_redirects = 3; /// What happens to a redirect: the client chases it (`max_redirects` hops), -/// `consume` receives it to walk the chain itself, or it fails the request. +/// `consume` receives it because `HeadChain` walks the chain itself, or it +/// fails the request. pub const Redirects = enum { follow, surface, @@ -263,7 +264,7 @@ pub fn exchange( ) anyerror!Outcome(T) { const spec = attempt.spec; // A followed redirect keeps each hop's Location and merged path here, - // 8 KiB of each; `req.uri` points into it. + // 8 KiB of each, as much as a `HeadChain` URL; `req.uri` points into it. var redirect_buffer: [2 * max_redirects * 8 * 1024]u8 = undefined; var req = attempt.governor.client.request(options.method, uri, .{ .redirect_behavior = options.redirects.behavior(), @@ -395,6 +396,85 @@ fn fatal(attempt: Attempt, stage: []const u8, err: anyerror) error{RequestFailed return error.RequestFailed; } +/// The URLs of a HEAD redirect chain walked by hand: std hands a HEAD its +/// redirect unfollowed. Three buffers in rotation: the current URL, the +/// `Location` it answered (copied out of a head that dies with the +/// response) with the path merged against it, and the resolved next URL. +pub const HeadChain = struct { + urls: [3][8 * 1024]u8 = undefined, + + pub const End = struct { + /// `start_url` itself, or a URL in the chain's buffers. + url: []const u8, + /// Its `Content-Length`, when it sent one. + size: ?u64, + }; + + /// Follows `start_url` through at most `max_hops` redirects, absolute + /// or relative, to the URL that answers a 2xx. Each hop is one governed + /// HEAD, retried as a GET would be. `authorize` names the credentials + /// each URL is sent. An absent or denied object fails the walk + /// (`ExchangeOptions.open`). + pub fn resolve( + self: *HeadChain, + governor: *Governor, + target: []const u8, + start_url: []const u8, + max_hops: usize, + context: anytype, + comptime authorize: fn (@TypeOf(context), url: []const u8) std.http.Client.Request.Headers.Value, + ) !End { + const Hop = struct { + context: @TypeOf(context), + url: []const u8, + uri: std.Uri, + /// Where a redirect's `Location` is copied. + location: []u8, + + const Result = union(enum) { end: ?u64, redirect: usize }; + + fn attempt(hop: *@This(), current: Attempt) anyerror!Outcome(Result) { + return exchange(Result, current, hop.uri, .{ + .method = .HEAD, + .authorization = authorize(hop.context, hop.url), + .open = true, + .redirects = .surface, + }, hop, consume); + } + + fn consume(hop: *@This(), res: *std.http.Client.Response) anyerror!Result { + if (res.head.status.class() == .success) return .{ .end = res.head.content_length }; + const location = res.head.location orelse return error.HttpRedirectLocationMissing; + if (location.len > hop.location.len) return error.HttpRedirectLocationOversize; + @memcpy(hop.location[0..location.len], location); + return .{ .redirect = location.len }; + } + }; + + var url = start_url; + // `start_url` is in none of the buffers. + var current: usize = self.urls.len - 1; + var hops: usize = 0; + while (true) { + const uri: std.Uri = try .parse(url); + const scratch = (current + 1) % self.urls.len; + var hop: Hop = .{ .context = context, .url = url, .uri = uri, .location = &self.urls[scratch] }; + const location_len = switch (try perform(Hop.Result, governor, .{ .target = target }, &hop, Hop.attempt)) { + .end => |size| return .{ .url = url, .size = size }, + .redirect => |len| len, + }; + + if (hops == max_hops) return error.TooManyHttpRedirects; + hops += 1; + const output = (current + 2) % self.urls.len; + var aux: []u8 = &self.urls[scratch]; + const resolved = try uri.resolveInPlace(location_len, &aux); + url = try std.fmt.bufPrint(&self.urls[output], "{f}", .{resolved}); + current = output; + } + } +}; + /// Retry classification of a non-2xx status; null when the status is not /// retried. `unavailable` is what a 503 means for the backend: rate limiting /// on the object stores (AWS `SlowDown`, GCS), a server failure elsewhere. From 4ebe32d03de9d6adde48d8ba3e090bdd9edfd3c1 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:02:11 +0100 Subject: [PATCH 036/103] vfs/http: move HTTP onto the governed loop; HTTP.init takes options HTTP's reads and lookups now run through request.zig. HTTP.init takes Options{protocol, retry} instead of a bare protocol, and its governor, named http, treats a 503 as a server failure. fetchSize walks redirects with HeadChain, which follows the relative Location that Hugging Face answers a resolve URL with, where HTTP's own walk failed. A positional read is one range_read GET that follows redirects and fills every buffer; it filled only the first. Opening, stat-ing and probing a path follow the open error contract, and access sends a HEAD where it succeeded for any path. A failed positional read is Canceled or Unexpected and a failed streaming read InputOutput, each logged once. A seek saturates at 0 instead of overflowing, and deinit logs the read totals at debug. Remotes passes .{ .protocol = .https }. VFSBase's io is aligned for the i96 fields of the governor HTTP now embeds, so recovering a backend from its base needs no @alignCast. HTTP has no parallel_read pool, so master's loader still reads https:// with one request per read. This breaks HTTP.init callers; llmd has none once it uses VFS.Remotes. Co-Authored-By: Claude Opus 5.5 --- vfs/base.zig | 5 +- vfs/http.zig | 170 ++++++-------------- vfs/http_acceptance_test.zig | 297 +++++++++++++++++++++++++++++++++++ vfs/request.zig | 8 +- vfs/vfs.zig | 21 ++- 5 files changed, 367 insertions(+), 134 deletions(-) diff --git a/vfs/base.zig b/vfs/base.zig index 9af3df66..fc21c0b7 100644 --- a/vfs/base.zig +++ b/vfs/base.zig @@ -41,7 +41,10 @@ pub const Backend = struct { }; pub const VFSBase = struct { - inner: std.Io, + /// Aligned like the i96 durations and timestamps a backend holds (its + /// `request.Governor`), so `@fieldParentPtr("base", ...)` needs no + /// `@alignCast`. + inner: std.Io align(@alignOf(std.Io.Duration)), pub fn init(io: std.Io) VFSBase { return .{ .inner = io }; diff --git a/vfs/http.zig b/vfs/http.zig index b1bc7b20..98b3fd85 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -4,10 +4,17 @@ const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; const Backend = @import("base.zig").Backend; +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const log = std.log.scoped(.@"zml/vfs/http"); pub const HTTP = struct { + pub const Options = struct { + protocol: Protocol, + retry: request.Retry = .{}, + }; + const Handle = struct { pub const Type = enum { file, @@ -40,22 +47,23 @@ pub const HTTP = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, - client: *std.http.Client, protocol: Protocol, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, protocol: Protocol) !HTTP { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !HTTP { return .{ .allocator = allocator, .base = .init(inner), - .client = http_client, - .protocol = protocol, + .protocol = options.protocol, + .governor = .init(inner, http_client, "http", .server_failure, options.retry), }; } pub fn deinit(self: *HTTP) void { + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -152,9 +160,9 @@ pub const HTTP = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.InputOutput }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -201,7 +209,9 @@ pub const HTTP = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); return .{ .inode = @intCast(0), @@ -216,15 +226,19 @@ pub const HTTP = struct { }; } - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + } fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; - var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -290,20 +304,18 @@ pub const HTTP = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); + // Saturates at both ends; `@abs` takes `minInt(i64)` too. handle.pos = if (relative_offset >= 0) - handle.pos + @as(u64, @intCast(relative_offset)) + handle.pos +| @abs(relative_offset) else - handle.pos - @as(u64, @intCast(-relative_offset)); + handle.pos -| @abs(relative_offset); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { @@ -319,117 +331,31 @@ pub const HTTP = struct { return path.len; } - fn fetchSize(self: *HTTP, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; - var redirect_buffer: [8 * 1024]u8 = undefined; - var aux_buffer: []u8 = &redirect_buffer; + /// The size of the object at `url` (a resolved path without its scheme), + /// from a HEAD of the end of its redirect chain, as far as a read's GET + /// follows it (`request.max_redirects`). + fn fetchSize(self: *HTTP, url: []const u8) !u64 { + var url_buffer: [8 * 1024]u8 = undefined; + const full_url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), url }); + var chain: request.HeadChain = .{}; + const end = try chain.resolve(&self.governor, url, full_url, request.max_redirects, {}, urlAuthorization); + return end.size orelse error.MissingContentLength; + } - const scheme = @tagName(self.protocol); - const url = try self.resolvePath(dir, sub_path, &path_buffer); - const full_url = try std.fmt.bufPrint(aux_buffer, "{s}://{s}", .{ scheme, url }); - - var uri = std.Uri.parse(full_url) catch return std.Io.File.OpenError.BadPathName; - while (true) { - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var res = try req.receiveHead(&redirect_buffer); - - switch (res.head.status.class()) { - .server_error, .client_error => { - log.err("Failed to fetch tree size for {s}", .{url}); - log.err("{s}", .{res.head.bytes}); - return error.ServerError; - }, - .informational => return error.UnexpectedStatus, - .success => return res.head.content_length.?, - .redirect => { - const location = res.head.location.?; - @memcpy(aux_buffer[0..location.len], location); - uri = uri.resolveInPlace(location.len, &aux_buffer) catch unreachable; - continue; - }, - } - } + /// Only what the URL carries: std's basic authorization from its + /// userinfo, as on every GET. + fn urlAuthorization(_: void, _: []const u8) std.http.Client.Request.Headers.Value { + return .default; } fn performRead(self: *HTTP, handle: *Handle, data: []const []u8, offset: u64) !usize { - if (offset >= handle.size) return 0; - - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += @as(u64, buf.len); - } - const remaining = handle.size - offset; - const take = @min(remaining, total_bytes); - const end = offset + take - 1; - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; - }; - var url_buffer: [8 * 1024]u8 = undefined; const url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), handle.uri }); const uri: std.Uri = try .parse(url); - - var req = try self.client.request(.GET, uri, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to perform read for {s}", .{handle.uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - - if (content_range) |cr| { - if (cr.start < offset) { - try reader.discardAll(offset - cr.start); - } - } - - return try reader.readSliceShort(data[0]); - } - - const ContentRange = struct { - start: u64, - end: u64, - total: u64, - }; - - fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; + return range_read.performRangeRead(&self.governor, .{ + .request = .{ .target = url }, + // The object `fetchSize` found at the end of the chain. + .redirects = .follow, + }, &uri, range_read.static, data, offset, handle.size); } }; diff --git a/vfs/http_acceptance_test.zig b/vfs/http_acceptance_test.zig index 5f68c410..8b78f662 100644 --- a/vfs/http_acceptance_test.zig +++ b/vfs/http_acceptance_test.zig @@ -1,5 +1,6 @@ const std = @import("std"); +const HTTP = @import("http.zig").HTTP; const MockServer = @import("mock_server.zig").MockServer; const range_read = @import("range_read.zig"); const vfs_request = @import("request.zig"); @@ -14,6 +15,221 @@ fn instantRetries(max_retries: usize) vfs_request.Retry { }; } +/// A `MockServer` serving `object`, an HTTP backend with a client of its +/// own, and `/object` open through that backend. Initialized in place: the +/// backend points at the client and the server must not move once started. +const Fixture = struct { + server: MockServer, + client: std.http.Client, + http: HTTP, + file: std.Io.File, + + fn init(self: *Fixture, object: []const u8, server_opts: MockServer.Options, retry: vfs_request.Retry) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + self.server = try .init(io, object, server_opts); + errdefer self.server.deinit(io); + try self.server.start(io); + self.client = .{ .allocator = allocator, .io = io }; + errdefer self.client.deinit(); + self.http = try .init(allocator, io, &self.client, .{ .protocol = .http, .retry = retry }); + errdefer self.http.deinit(); + var path_buffer: [128]u8 = undefined; + const path = try self.server.path(&path_buffer); + self.file = try std.Io.Dir.openFile(.cwd(), self.http.io(), path, .{ .mode = .read_only }); + } + + fn deinit(self: *Fixture) void { + self.file.close(self.http.io()); + self.http.deinit(); + self.client.deinit(); + self.server.deinit(std.testing.io); + } +}; + +/// Reads `object` back through the fixture's file with `readers` concurrent +/// positional reads of `read_size` bytes, reader `i` at `i * read_size`, +/// all released at once. Fails on the first reader that failed or read +/// other bytes. +fn expectConcurrentReads(fixture: *Fixture, object: []const u8, comptime readers: usize, comptime read_size: usize) !void { + const Reader = struct { + fn run(file: std.Io.File, file_io: std.Io, output: []u8, offset: u64, result: *anyerror!usize, start: *std.Io.Event) void { + start.wait(file_io) catch |err| { + result.* = err; + return; + }; + result.* = file.readPositional(file_io, &.{output}, offset); + } + }; + const io = std.testing.io; + var outputs: [readers][read_size]u8 = undefined; + var results: [readers]anyerror!usize = undefined; + var released: std.Io.Event = .unset; + var group: std.Io.Group = .init; + defer group.cancel(io); + for (&outputs, &results, 0..) |*output, *result, index| { + try group.concurrent(io, Reader.run, .{ fixture.file, fixture.http.io(), output, index * read_size, result, &released }); + } + released.set(io); + try group.await(io); + + for (&outputs, results, 0..) |*output, result, index| { + try std.testing.expectEqual(read_size, try result); + try std.testing.expectEqualSlices(u8, object[index * read_size ..][0..read_size], output); + } +} + +test "generic HTTP keeps a large scattered positional read to one GET" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const offset = 13; + const request_size = 17 * 1024 * 1024 + 257; + + const object = try allocator.alloc(u8, offset + request_size); + defer allocator.free(object); + @memset(object, 0xa5); + const output = try allocator.alloc(u8, request_size); + defer allocator.free(output); + + var fixture: Fixture = undefined; + try fixture.init(object, .{}, .{}); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + // Fresh stats for the read: the open's HEAD is a request too. + stats.* = .{}; + const buffers = [_][]u8{ + output[0 .. 4 * 1024 * 1024], + output[4 * 1024 * 1024 .. 10 * 1024 * 1024], + output[10 * 1024 * 1024 ..], + }; + try std.testing.expectEqual( + request_size, + try fixture.file.readPositional(fixture.http.io(), &buffers, offset), + ); + try std.testing.expectEqualSlices(u8, object[offset..], output); + + try server.stop(io); + + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); +} + +test "generic HTTP follows a relative redirect chain at open and on every read" { + const io = std.testing.io; + + var object: [4096]u8 = undefined; + for (&object, 0..) |*byte, i| byte.* = @truncate(i); + var output: [1000]u8 = undefined; + + // `/object` redirects twice, and the second location resolves against + // the first hop's path. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .redirects = &.{ + .{ .path = "/object", .location = "/hop/next" }, + .{ .path = "/hop/next", .location = "../moved" }, + }, + .objects = &.{.{ .path = "/moved", .bytes = &object }}, + }, .{}); + defer fixture.deinit(); + const server = &fixture.server; + const http_io = fixture.http.io(); + + try std.testing.expectEqual(@as(u64, object.len), (try fixture.file.stat(http_io)).size); + const buffers = [_][]u8{&output}; + try std.testing.expectEqual(output.len, try fixture.file.readPositional(http_io, &buffers, 77)); + try std.testing.expectEqualSlices(u8, object[77..][0..output.len], &output); + + try server.stop(io); + // Only the object answers with bytes; the hops before it redirect. + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.get_requests.load(.acquire)); +} + +test "generic HTTP retries serially and reports aggregate retry counters" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const request_size = 2 * 1024 * 1024; + + const object = try allocator.alloc(u8, request_size); + defer allocator.free(object); + @memset(object, 0x3c); + const output = try allocator.alloc(u8, request_size); + defer allocator.free(output); + + var fixture: Fixture = undefined; + try fixture.init(object, .{ .fail_first_gets = 1 }, .{ + .max_retries = 1, + .initial_delay = .fromNanoseconds(0), + .max_delay = .fromNanoseconds(0), + }); + defer fixture.deinit(); + const server = &fixture.server; + const http_io = fixture.http.io(); + const stats = &fixture.http.governor.stats; + + stats.* = .{}; + const one_buffer = [_][]u8{output}; + try std.testing.expectEqual( + request_size, + try fixture.file.readPositional(http_io, &one_buffer, 0), + ); + try std.testing.expectEqual(@as(u64, 2), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); + + stats.* = .{}; + try std.testing.expectEqual( + request_size, + try fixture.file.readPositional(http_io, &one_buffer, 0), + ); + try std.testing.expectEqual(@as(u64, 1), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expectEqualSlices(u8, object, output); + + try server.stop(io); + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); +} + +test "generic HTTP physical concurrency does not exceed caller admission" { + const io = std.testing.io; + const admission = 3; + const read_size = 1024; + + var object: [admission * read_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index); + + var fixture: Fixture = undefined; + try fixture.init(&object, .{ .barrier_gets = admission }, .{}); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + stats.* = .{}; + try expectConcurrentReads(&fixture, &object, admission, read_size); + + try server.stop(io); + + const peak = server.peak_gets.load(.acquire); + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, admission), peak); + try std.testing.expect(peak <= admission); + try std.testing.expectEqual( + @as(usize, admission), + server.get_requests.load(.acquire), + ); + try std.testing.expectEqual(@as(u64, admission), stats.physical_requests.load(.acquire)); +} + test "the shared range loop prepares the request once per attempt" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -77,3 +293,84 @@ test "the shared range loop prepares the request once per attempt" { try std.testing.expectEqual(@as(u64, 1), stats.server_failures.load(.acquire)); try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); } + +test "http fails open, stat and access of an absent or denied object" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Case = struct { name: []const u8, status: std.http.Status, expected: anyerror }; + const cases = [_]Case{ + .{ .name = "missing", .status = .not_found, .expected = error.FileNotFound }, + .{ .name = "gone", .status = .gone, .expected = error.FileNotFound }, + .{ .name = "unauthorized", .status = .unauthorized, .expected = error.PermissionDenied }, + .{ .name = "forbidden", .status = .forbidden, .expected = error.PermissionDenied }, + }; + const refusals = comptime refusals: { + var list: [cases.len]MockServer.Refusal = undefined; + for (cases, 0..) |case, i| list[i] = .{ .path = "/" ++ case.name, .status = case.status }; + break :refusals list; + }; + + var object: [32]u8 = undefined; + @memset(&object, 0x6b); + var server = try MockServer.init(io, &object, .{ .refusals = &refusals }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http }); + defer http.deinit(); + const http_io = http.io(); + + var path_buffer: [128]u8 = undefined; + const present = try server.path(&path_buffer); + try std.Io.Dir.access(.cwd(), http_io, present, .{}); + try std.testing.expectEqual(@as(u64, object.len), (try std.Io.Dir.statFile(.cwd(), http_io, present, .{})).size); + for (cases) |case| { + const path = try std.fmt.bufPrint(&path_buffer, "127.0.0.1:{d}/{s}", .{ server.port(), case.name }); + try std.testing.expectError(case.expected, std.Io.Dir.openFile(.cwd(), http_io, path, .{})); + try std.testing.expectError(case.expected, std.Io.Dir.statFile(.cwd(), http_io, path, .{})); + try std.testing.expectError(case.expected, std.Io.Dir.access(.cwd(), http_io, path, .{})); + } + + try server.stop(io); +} + +test "http fails a refused streaming read with InputOutput and seeks back no further than 0" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + var object: [64]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 5 +% 1); + // The first GET is answered 500, with no retry left. + var server = try MockServer.init(io, &object, .{ .fail_first_gets = 1 }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(0) }); + defer http.deinit(); + const http_io = http.io(); + + var path_buffer: [128]u8 = undefined; + const file = try std.Io.Dir.openFile(.cwd(), http_io, try server.path(&path_buffer), .{}); + defer file.close(http_io); + + var head: [8]u8 = undefined; + const head_slices = [_][]u8{&head}; + try std.testing.expectError(error.InputOutput, file.readStreaming(http_io, &head_slices)); + // Forward, then back by more than the position: at 0. + try http_io.vtable.fileSeekBy(http_io.userdata, file, 5); + try http_io.vtable.fileSeekBy(http_io.userdata, file, -9); + try std.testing.expectEqual(head.len, try file.readStreaming(http_io, &head_slices)); + try std.testing.expectEqualSlices(u8, object[0..head.len], &head); + // `minInt(i64)`, whose negation overflows, stops at 0 too. + try http_io.vtable.fileSeekBy(http_io.userdata, file, std.math.minInt(i64)); + try std.testing.expectEqual(head.len, try file.readStreaming(http_io, &head_slices)); + try std.testing.expectEqualSlices(u8, object[0..head.len], &head); + + try server.stop(io); + try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); +} diff --git a/vfs/request.zig b/vfs/request.zig index 79351cce..96769665 100644 --- a/vfs/request.zig +++ b/vfs/request.zig @@ -487,11 +487,9 @@ fn classifyStatus(status: std.http.Status, unavailable: ReadFailure) ?ReadFailur }; } -/// What an open, stat or access of a remote path fails with: `FileNotFound` -/// for a missing object (404, 410), `PermissionDenied` for a denied one -/// (401, 403), `Canceled` for a canceled task and `SystemResources` when a -/// request cannot allocate. `openStatus` maps the statuses; anything else is -/// `Unexpected`, logged once, by `exchange` or `Governor.openFailure`. +/// What an open, stat or access of a remote path fails with (vfs.zig's +/// header); `openStatus` maps the statuses, and anything else is logged +/// once, by `exchange` or `Governor.openFailure`. pub const OpenError = error{ FileNotFound, PermissionDenied, Canceled, SystemResources, Unexpected }; /// The error a status means for the path an open request diff --git a/vfs/vfs.zig b/vfs/vfs.zig index 6d39dc4c..557eaad2 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -8,11 +8,20 @@ //! The VFS, and every backend registered on it, must outlive each file //! opened through it. //! -//! A remote path is read-only: creating or deleting a file or a directory -//! there, or opening a file for writing, is logged at err and fails with -//! `ReadOnlyFileSystem`. A local file fails as the inner io does (a denied -//! one is `AccessDenied` there); it may be read past the page cache -//! (`useDirectIo`). +//! Errors of a remote path on a backend that runs its requests through +//! `request.zig` (`HTTP`, which serves `https://`): opening, stat-ing or +//! probing it (`access`) fails with `FileNotFound` for a missing object +//! (404, 410), `PermissionDenied` for a denied one (401, 403), `Canceled` +//! for a canceled task, and `SystemResources` when a request cannot +//! allocate. Any other failure of a request is logged at err with the +//! backend and the path, and surfaces as `Unexpected`. A positional read +//! that still fails after its retries is logged at err and surfaces as +//! `Unexpected`; a streaming read as `InputOutput`. A remote path is +//! read-only: creating or deleting a file or a directory there, or opening +//! a file for writing, is logged at err and fails with `ReadOnlyFileSystem`. +//! A local file fails as the inner io does (a denied one is `AccessDenied` +//! there); it may be read past the page cache (`useDirectIo`). A backend on +//! `request.zig` logs its read totals at debug from its `deinit`. const std = @import("std"); const stdx = @import("stdx"); @@ -132,7 +141,7 @@ pub const Remotes = struct { self.client = .{ .allocator = allocator, .io = base_io }; errdefer self.client.deinit(); try self.client.initDefaultProxies(self.arena.allocator(), environ); - self.https = try .init(allocator, base_io, &self.client, .https); + self.https = try .init(allocator, base_io, &self.client, .{ .protocol = .https }); errdefer self.https.deinit(); self.hf = try .auto(allocator, base_io, &self.client, environ); errdefer self.hf.deinit(); From 04cd4f7ab3563a9af5ddde9d03e6f5b1c3168a01 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:13:59 +0100 Subject: [PATCH 037/103] vfs: retry a stalled exchange A peer that stops sending without closing its connection holds the reader waiting on it, and the load waiting on that reader, forever. Each exchange now runs on a task of its own while the attempt watches it: range_read counts the body bytes it receives in Attempt.progress, the attempt checks that count twelve times per Retry.stall_timeout (60 s by default), and after a whole timeout without progress it cancels the exchange and retries it as a timeout. An exchange that had already finished when it was canceled keeps its value. Where the io cannot run the exchange concurrently, it runs unwatched. A request that a hook sends before its exchange, such as a re-resolve, is watched by its own exchange, so waiting on it is no stall. Co-Authored-By: Claude Opus 5.5 --- vfs/http_acceptance_test.zig | 32 ++++++ vfs/range_read.zig | 45 ++++++--- vfs/request.zig | 187 +++++++++++++++++++++++++++++++++-- 3 files changed, 245 insertions(+), 19 deletions(-) diff --git a/vfs/http_acceptance_test.zig b/vfs/http_acceptance_test.zig index 8b78f662..30847273 100644 --- a/vfs/http_acceptance_test.zig +++ b/vfs/http_acceptance_test.zig @@ -294,6 +294,38 @@ test "the shared range loop prepares the request once per attempt" { try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); } +test "generic HTTP cancels a GET that receives nothing and retries it as a timeout" { + const io = std.testing.io; + + var object: [4096]u8 = undefined; + for (&object, 0..) |*byte, i| byte.* = @truncate(i); + var output: [1000]u8 = undefined; + + // The first GET waits unanswered for the second: a peer that receives + // the request and sends nothing. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ .barrier_gets = 2 }, .{ + .initial_delay = .fromNanoseconds(0), + .max_delay = .fromNanoseconds(0), + .stall_timeout = .fromMilliseconds(100), + }); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + stats.* = .{}; + const buffers = [_][]u8{&output}; + try std.testing.expectEqual(output.len, try fixture.file.readPositional(fixture.http.io(), &buffers, 77)); + try std.testing.expectEqualSlices(u8, object[77..][0..output.len], &output); + try std.testing.expectEqual(@as(u64, 1), stats.timeouts.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); + + // The first GET answers late, to a connection the client closed: that + // write may fail. + server.stop(io) catch |err| if (err != error.WriteFailed) return err; + try std.testing.expectEqual(@as(usize, 2), server.get_requests.load(.acquire)); +} + test "http fails open, stat and access of an absent or denied object" { const allocator = std.testing.allocator; const io = std.testing.io; diff --git a/vfs/range_read.zig b/vfs/range_read.zig index e2ce3ff0..9ab72d91 100644 --- a/vfs/range_read.zig +++ b/vfs/range_read.zig @@ -67,8 +67,10 @@ pub fn performRangeRead( data: []const []u8, offset: u64, size: usize, + progress: *std.atomic.Value(u64) = undefined, fn attempt(self: *@This(), current: request.Attempt) anyerror!request.Outcome(void) { + self.progress = current.progress; const prepared = try prepare(self.context, current, self.range); std.debug.assert(prepared.extra_headers.len <= max_extra_headers); var headers: [max_extra_headers + 1]std.http.Header = undefined; @@ -84,7 +86,7 @@ pub fn performRangeRead( fn consume(self: *@This(), res: *std.http.Client.Response) anyerror!void { // The head bytes are released when the body reader is taken. const content_range = contentRange(res.head); - return readResponse(res.reader(&.{}), res.head.status, content_range, self.offset, self.data, self.size); + return readResponse(res.reader(&.{}), res.head.status, content_range, self.offset, self.data, self.size, self.progress); } }; @@ -144,7 +146,8 @@ fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { } /// Validates the response against the requested range, discards the prefix -/// of a `200` that ignored `Range`, then scatters `read_size` bytes. +/// of a `200` that ignored `Range`, then scatters `read_size` bytes, counting +/// what it receives in `progress`. fn readResponse( reader: *std.Io.Reader, status: std.http.Status, @@ -152,6 +155,7 @@ fn readResponse( offset: u64, data: []const []u8, read_size: usize, + progress: ?*std.atomic.Value(u64), ) !void { std.debug.assert(read_size != 0); @@ -166,16 +170,34 @@ fn readResponse( else => return error.UnexpectedStatus, }; - try reader.discardAll64(offset - response_start); - try readScatter(reader, data, read_size); + var skip = offset - response_start; + while (skip != 0) { + const len = @min(skip, progress_slice); + try reader.discardAll(len); + count(progress, len); + skip -= len; + } + try readScatter(reader, data, read_size, progress); } -fn readScatter(reader: *std.Io.Reader, data: []const []u8, len: usize) !void { +/// Bytes read between two progress counts. +const progress_slice = 1 << 20; + +fn count(progress: ?*std.atomic.Value(u64), len: u64) void { + if (progress) |counter| _ = counter.fetchAdd(len, .monotonic); +} + +fn readScatter(reader: *std.Io.Reader, data: []const []u8, len: usize, progress: ?*std.atomic.Value(u64)) !void { var remaining = len; for (data) |buf| { - const destination = buf[0..@min(remaining, buf.len)]; + var destination = buf[0..@min(remaining, buf.len)]; remaining -= destination.len; - try reader.readSliceAll(destination); + while (destination.len != 0) { + const slice = destination[0..@min(destination.len, progress_slice)]; + try reader.readSliceAll(slice); + count(progress, slice.len); + destination = destination[slice.len..]; + } } if (remaining != 0) return error.UnexpectedEndOfOutput; } @@ -207,6 +229,7 @@ test "range responses fill scatter buffers" { 3, &.{ &first, &second }, 5, + null, ); try std.testing.expectEqualStrings("34", &first); try std.testing.expectEqualStrings("567", &second); @@ -216,7 +239,7 @@ test "200 responses that ignore Range are positioned and scattered" { var reader: std.Io.Reader = .fixed("0123456789"); var first: [1]u8 = undefined; var second: [4]u8 = undefined; - try readResponse(&reader, .ok, null, 3, &.{ &first, &second }, 5); + try readResponse(&reader, .ok, null, 3, &.{ &first, &second }, 5, null); try std.testing.expectEqualStrings("3", &first); try std.testing.expectEqualStrings("4567", &second); } @@ -224,17 +247,17 @@ test "200 responses that ignore Range are positioned and scattered" { test "partial responses require a covering Content-Range" { var reader: std.Io.Reader = .fixed("3456"); var output: [4]u8 = undefined; - try std.testing.expectError(error.InvalidContentRange, readResponse(&reader, .partial_content, null, 3, &.{&output}, 4)); + try std.testing.expectError(error.InvalidContentRange, readResponse(&reader, .partial_content, null, 3, &.{&output}, 4, null)); reader = .fixed("4567"); try std.testing.expectError( error.InvalidContentRange, - readResponse(&reader, .partial_content, .{ .start = 4, .end = 7, .total = 10 }, 3, &.{&output}, 4), + readResponse(&reader, .partial_content, .{ .start = 4, .end = 7, .total = 10 }, 3, &.{&output}, 4, null), ); reader = .fixed("345"); try std.testing.expectError( error.InvalidContentRange, - readResponse(&reader, .partial_content, .{ .start = 3, .end = 5, .total = 10 }, 3, &.{&output}, 4), + readResponse(&reader, .partial_content, .{ .start = 3, .end = 5, .total = 10 }, 3, &.{&output}, 4, null), ); } diff --git a/vfs/request.zig b/vfs/request.zig index 96769665..4b006aad 100644 --- a/vfs/request.zig +++ b/vfs/request.zig @@ -16,6 +16,13 @@ pub const Retry = struct { /// Longest backoff before a request's next attempt, a server-named delay /// included. max_delay: std.Io.Duration = .fromSeconds(30), + /// An exchange that receives nothing for this long is canceled and + /// retried as a timeout: a peer that stops sending without closing the + /// connection would otherwise hold its reader, and the load waiting for + /// it, forever. Body readers count what they receive + /// (`Attempt.progress`); an exchange that counts nothing (a HEAD, a token + /// exchange) must finish within it. + stall_timeout: std.Io.Duration = .fromSeconds(60), }; /// One per backend instance: what each of its requests needs and its read @@ -137,8 +144,15 @@ pub const Attempt = struct { /// The status the previous attempt returned, for a hook that reacts to /// one (an expired signed URL answering 403). previous_status: ?std.http.Status = null, + /// Bytes received so far, for a hook whose body is long: `exchange` + /// cancels an exchange this counter does not move for + /// `Retry.stall_timeout`. + progress: *std.atomic.Value(u64), }; +/// How often a watched exchange's progress is checked, per stall timeout. +const stall_checks = 12; + /// Identity of one request, for the logs and for the classification. pub const RequestSpec = struct { /// Object identity for log lines. @@ -174,9 +188,10 @@ pub fn Outcome(comptime T: type) type { /// The one request loop. `attemptFn` performs a single HTTP exchange (see /// `exchange`); this drives the classification and the retry budget around -/// it. Every retryable failure, a throttle included, sleeps the per-request -/// backoff, or the delay the server named up to `max_delay`, and charges one -/// of `max_retries`. A cancel of the caller propagates. +/// it. Every retryable failure, a throttle and a stall +/// (`Retry.stall_timeout`) included, sleeps the per-request backoff, or the +/// delay the server named up to `max_delay`, and charges one of +/// `max_retries`. A cancel of the caller propagates. pub fn perform( comptime T: type, governor: *Governor, @@ -188,7 +203,8 @@ pub fn perform( var previous_status: ?std.http.Status = null; while (true) { governor.stats.recordAttempt(); - switch (try attemptFn(context, .{ .governor = governor, .spec = spec, .previous_status = previous_status })) { + var progress: std.atomic.Value(u64) = .init(0); + switch (try attemptFn(context, .{ .governor = governor, .spec = spec, .previous_status = previous_status, .progress = &progress })) { .done => |value| return value, .retry => |failure| { governor.stats.recordFailure(failure.failure); @@ -208,6 +224,74 @@ pub fn perform( } } +/// Runs one exchange on its own unit of concurrency while this task watches +/// the attempt's progress, and cancels it after `Retry.stall_timeout` without +/// progress. The exchange returns a canceled read as `Canceled`, unlogged, +/// so the outcome of a stalled exchange is decided here, as a timeout. +/// Without a unit of concurrency the exchange runs unwatched. +fn watched( + comptime T: type, + context: anytype, + comptime exchangeFn: fn (@TypeOf(context), Attempt) anyerror!Outcome(T), + attempt: Attempt, +) anyerror!Outcome(T) { + const Run = struct { + result: anyerror!Outcome(T) = undefined, + done: std.Io.Event = .unset, + + fn run(self: *@This(), io: std.Io, context_: @TypeOf(context), attempt_: Attempt) void { + self.result = exchangeFn(context_, attempt_); + self.done.set(io); + } + }; + const governor = attempt.governor; + var run: Run = .{}; + var future = governor.io.concurrent(Run.run, .{ &run, governor.io, context, attempt }) catch |err| switch (err) { + error.ConcurrencyUnavailable => return exchangeFn(context, attempt), + }; + const stall_timeout = governor.retry.stall_timeout; + const check: std.Io.Timeout = .{ .duration = .{ + .raw = .fromNanoseconds(@divTrunc(stall_timeout.nanoseconds, stall_checks)), + .clock = .awake, + } }; + var seen = attempt.progress.load(.monotonic); + var quiet: usize = 0; + while (quiet < stall_checks) { + run.done.waitTimeout(governor.io, check) catch |err| switch (err) { + error.Timeout => { + const now = attempt.progress.load(.monotonic); + quiet = if (now == seen) quiet + 1 else 0; + seen = now; + continue; + }, + error.Canceled => { + future.cancel(governor.io); + // An exchange already past its last cancelation point + // finished: its value is the caller's, and the cancel + // reaches the caller at its next cancelation point. + if (run.result) |outcome| switch (outcome) { + .done => { + governor.io.recancel(); + return outcome; + }, + .retry => {}, + } else |_| {} + return error.Canceled; + }, + }; + future.await(governor.io); + return run.result; + } + future.cancel(governor.io); + // It may have finished while being canceled. + if (run.result) |outcome| switch (outcome) { + .done => return outcome, + .retry => {}, + } else |_| {} + log.warn("{s}: {s} received nothing for {d:.1}s; retrying", .{ governor.name, attempt.spec.target, @as(f64, @floatFromInt(stall_timeout.nanoseconds)) / std.time.ns_per_s }); + return .{ .retry = .{ .failure = .timeout } }; +} + fn clampDelay(value: std.Io.Duration, low: std.Io.Duration, high: std.Io.Duration) std.Io.Duration { return .fromNanoseconds(std.math.clamp(value.nanoseconds, low.nanoseconds, high.nanoseconds)); } @@ -253,7 +337,9 @@ pub const ExchangeOptions = struct { /// an unclassified status or any other connect or head failure fails with /// `error.RequestFailed`, logged at err, except that an open request /// (`ExchangeOptions.open`) fails an absent or denied object with -/// `FileNotFound` or `PermissionDenied` first, unlogged. +/// `FileNotFound` or `PermissionDenied` first, unlogged. The exchange alone +/// is watched (`Retry.stall_timeout`); a governed request its hook runs +/// first (a re-resolve, a token refresh) is watched by its own exchange. pub fn exchange( comptime T: type, attempt: Attempt, @@ -261,6 +347,26 @@ pub fn exchange( options: ExchangeOptions, context: anytype, comptime consume: fn (@TypeOf(context), *std.http.Client.Response) anyerror!T, +) anyerror!Outcome(T) { + const Exchange = struct { + uri: std.Uri, + options: ExchangeOptions, + context: @TypeOf(context), + + fn run(self: @This(), current: Attempt) anyerror!Outcome(T) { + return roundTrip(T, current, self.uri, self.options, self.context, consume); + } + }; + return watched(T, Exchange{ .uri = uri, .options = options, .context = context }, Exchange.run, attempt); +} + +fn roundTrip( + comptime T: type, + attempt: Attempt, + uri: std.Uri, + options: ExchangeOptions, + context: anytype, + comptime consume: fn (@TypeOf(context), *std.http.Client.Response) anyerror!T, ) anyerror!Outcome(T) { const spec = attempt.spec; // A followed redirect keeps each hop's Location and merged path here, @@ -284,7 +390,7 @@ pub fn exchange( error.NetworkDown, error.NameServerFailure, => return retryable(T, attempt, "connect", err, .transient), - // The caller's cancel. + // A stalled exchange that `watched` canceled, or the caller's cancel. error.Canceled => return err, error.OutOfMemory => return err, else => return fatal(attempt, "connect", err), @@ -329,7 +435,7 @@ pub fn exchange( error.NetworkDown, error.NameServerFailure, => return retryable(T, attempt, "receive headers", err, .transient), - // The caller's cancel. + // A stalled exchange that `watched` canceled, or the caller's cancel. error.Canceled => return err, error.OutOfMemory => return err, else => return fatal(attempt, "receive headers", err), @@ -656,6 +762,69 @@ test "a server-named retry delay sleeps at most max_delay" { try std.testing.expectEqual(@as(u64, 20 * std.time.ns_per_ms), governor.stats.retry_delay_ns.load(.acquire)); } +test "an attempt that receives nothing is canceled and retried as a timeout" { + var governor: Governor = .init(std.testing.io, undefined, "test", .throttle, .{ + .max_retries = 3, + .initial_delay = .fromMilliseconds(1), + .max_delay = .fromMilliseconds(1), + .stall_timeout = .fromMilliseconds(120), + }); + const Hang = struct { + seen: usize = 0, + progressed: u64 = 0, + // Watched as `exchange` watches its round trip. + fn attempt(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + return watched(usize, self, receive, current); + } + + fn receive(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + // The first exchange hangs, the second one keeps receiving and + // takes longer than the stall timeout overall. + if (self.seen == 1) { + try std.testing.io.sleep(.fromSeconds(60), .awake); + return error.TestUnexpectedResult; + } + for (0..6) |_| { + try std.testing.io.sleep(.fromMilliseconds(40), .awake); + _ = current.progress.fetchAdd(1, .monotonic); + self.progressed += 1; + } + return .{ .done = self.seen }; + } + }; + var hang: Hang = .{}; + try std.testing.expectEqual(@as(usize, 2), try perform(usize, &governor, test_spec, &hang, Hang.attempt)); + try std.testing.expectEqual(@as(u64, 6), hang.progressed); + try std.testing.expectEqual(@as(u64, 1), governor.stats.timeouts.load(.acquire)); +} + +test "an attempt waiting on a request of its own is not a stall" { + var governor: Governor = .init(std.testing.io, undefined, "test", .throttle, .{ + .max_retries = 3, + .initial_delay = .fromMilliseconds(1), + .max_delay = .fromMilliseconds(1), + .stall_timeout = .fromMilliseconds(120), + }); + const Nested = struct { + fn attempt(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + _ = try perform(usize, current.governor, test_spec, self, wait); + return .{ .done = 1 }; + } + + // A re-resolve or token refresh waiting out a hold, outside any + // exchange, for longer than the stall timeout. + fn wait(_: *@This(), _: Attempt) anyerror!Outcome(usize) { + try std.testing.io.sleep(.fromMilliseconds(300), .awake); + return .{ .done = 0 }; + } + }; + var nested: Nested = .{}; + try std.testing.expectEqual(@as(usize, 1), try perform(usize, &governor, test_spec, &nested, Nested.attempt)); + try std.testing.expectEqual(@as(u64, 0), governor.stats.timeouts.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), governor.stats.physical_requests.load(.acquire)); +} + test "an exchange canceled while it waits for the head returns Canceled, not a retry" { const MockServer = @import("mock_server.zig").MockServer; const io = std.testing.io; @@ -672,8 +841,10 @@ test "an exchange canceled while it waits for the head returns Canceled, not a r const uri: std.Uri = try .parse(try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()})); const Exchange = struct { + // Unwatched, so that the cancel lands in the read rather than in `watched`. fn run(governor_: *Governor, uri_: std.Uri, result: *anyerror!Outcome(void)) void { - result.* = exchange(void, .{ .governor = governor_, .spec = test_spec }, uri_, .{ + var progress: std.atomic.Value(u64) = .init(0); + result.* = roundTrip(void, .{ .governor = governor_, .spec = test_spec, .progress = &progress }, uri_, .{ .extra_headers = &.{.{ .name = "Range", .value = "bytes=0-5" }}, }, {}, consume); } From 307d17d44715ca57bdfa7d2fe363e80e0bc4b619 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:16:36 +0100 Subject: [PATCH 038/103] vfs: hold a throttled backend A throttle was a charged retry: each reader backed off on its own and retried, so a rate-limited backend kept seeing about as many requests and a long limit exhausted every reader's retries. A throttle (429 everywhere, 503 on a backend that names it so) now arms a hold over every request of the backend. Each request waits out the hold before it goes, waking at the deadline plus jitter of its own, and the wait is cancelable. A throttle arms the hold for the delay the server named or for a full-jitter backoff on the throttles of the episode, whichever is longer, clamped between the initial delay and max_hold (2 minutes); a later answer can extend a running hold but never shorten it. A clean window as long as the hold ends the episode, and an episode longer than throttle_budget (5 minutes) fails the backend's requests with RateLimited until one does. Throttles are no longer charged to max_retries. The governor counts holds and the time requests spent waiting them out. Requests carry their URI's authority as the hold's key, though one hold covers the whole backend. Tested against MockServer only; no real throttling source was available. Co-Authored-By: Claude Opus 5.5 --- vfs/http.zig | 2 +- vfs/http_acceptance_test.zig | 220 ++++++++++++++++++- vfs/range_read.zig | 6 +- vfs/request.zig | 397 ++++++++++++++++++++++++++++++++--- vfs/vfs.zig | 14 +- 5 files changed, 596 insertions(+), 43 deletions(-) diff --git a/vfs/http.zig b/vfs/http.zig index 98b3fd85..a9765d0f 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -353,7 +353,7 @@ pub const HTTP = struct { const url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), handle.uri }); const uri: std.Uri = try .parse(url); return range_read.performRangeRead(&self.governor, .{ - .request = .{ .target = url }, + .request = .{ .target = url, .key = request.authorityOf(uri) }, // The object `fetchSize` found at the end of the chain. .redirects = .follow, }, &uri, range_read.static, data, offset, handle.size); diff --git a/vfs/http_acceptance_test.zig b/vfs/http_acceptance_test.zig index 30847273..d79bfcbb 100644 --- a/vfs/http_acceptance_test.zig +++ b/vfs/http_acceptance_test.zig @@ -12,6 +12,19 @@ fn instantRetries(max_retries: usize) vfs_request.Retry { .max_retries = max_retries, .initial_delay = .fromNanoseconds(0), .max_delay = .fromNanoseconds(0), + .max_hold = .fromNanoseconds(0), + .throttle_budget = .fromSeconds(300), + }; +} + +/// A retry configuration whose retries and holds are short enough for a test. +fn fastRetry(initial_ms: i64, budget_ms: i64) vfs_request.Retry { + return .{ + .max_retries = 5, + .initial_delay = .fromMilliseconds(initial_ms), + .max_delay = .fromMilliseconds(initial_ms), + .max_hold = .fromSeconds(30), + .throttle_budget = .fromMilliseconds(budget_ms), }; } @@ -261,7 +274,9 @@ test "the shared range loop prepares the request once per attempt" { var url_buffer: [160]u8 = undefined; const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); var signed: Signed = .{ .uri = try .parse(url) }; - const spec: range_read.RangeSpec = .{ .request = .{ .target = url } }; + const spec: range_read.RangeSpec = .{ + .request = .{ .target = url, .key = vfs_request.authorityOf(signed.uri) }, + }; const buffers = [_][]u8{ output[0..1024], output[1024..] }; // No retry budget: one attempt, one hook call, the failure is counted. @@ -294,6 +309,209 @@ test "the shared range loop prepares the request once per attempt" { try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); } +test "one throttled GET holds every reader of the backend" { + const io = std.testing.io; + const readers = 8; + const read_size = 512; + + var object: [readers * read_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index); + + // Every reader's first GET is throttled, all eight answered together (the + // barrier), with a one second reset. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .barrier_gets = readers, + .throttle = .{ .first_gets = readers, .retry_after_s = 1 }, + }, fastRetry(50, 300_000)); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + try expectConcurrentReads(&fixture, &object, readers, read_size); + + try server.stop(io); + + // One hold covered every reader, the server-named second was honoured, + // and no reader burned a retry for it. + try std.testing.expectEqual(@as(u64, 1), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, readers), stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(usize, 2 * readers), server.get_requests.load(.acquire)); + const throttled_at = server.first_throttle_ns.load(.acquire); + const resumed_at = server.first_get_after_throttle_ns.load(.acquire); + try std.testing.expect(throttled_at != 0 and resumed_at != 0); + try std.testing.expect(resumed_at - throttled_at >= std.time.ns_per_s); +} + +test "a rate window is absorbed by the hold instead of the retry budget" { + const io = std.testing.io; + const readers = 12; + const read_size = 256; + + var object: [readers * read_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 3); + + // Four GETs per 100 ms; twelve readers need three windows at best. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .throttle = .{ .window = .{ .gets = 4, .per_ms = 100 } }, + }, fastRetry(50, 300_000)); + defer fixture.deinit(); + const stats = &fixture.http.governor.stats; + + try expectConcurrentReads(&fixture, &object, readers, read_size); + + try fixture.server.stop(io); + + try std.testing.expect(stats.holds.load(.acquire) > 0); + try std.testing.expect(stats.throttles.load(.acquire) > 0); + // A limited server never exhausts a reader's retries: the hold carries + // the whole episode. + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expect(stats.hold_wait_ns.load(.acquire) > 0); +} + +test "a throttled HEAD at open waits the hold out and succeeds" { + const io = std.testing.io; + + var object: [64]u8 = undefined; + @memset(&object, 0x11); + + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .throttle = .{ .first_heads = 1, .retry_after_s = 0 }, + }, fastRetry(20, 300_000)); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + try std.testing.expectEqual(@as(u64, object.len), try fixture.file.length(fixture.http.io())); + + try server.stop(io); + + // The open retried its HEAD after the hold, and the hold is the + // backend's, not the read path's. + try std.testing.expectEqual(@as(usize, 2), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); +} + +test "a 503 retries a plain HTTP server without holding it" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const read_size = 512; + + var object: [read_size]u8 = undefined; + @memset(&object, 0x7e); + var output: [read_size]u8 = undefined; + + // The first two GETs are answered 503. + var server = try MockServer.init(io, &object, .{ + .throttle = .{ .first_gets = 2, .status = .service_unavailable }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(5) }); + defer http.deinit(); + const http_io = http.io(); + + var path_buffer: [128]u8 = undefined; + const file = try std.Io.Dir.openFile(.cwd(), http_io, try server.path(&path_buffer), .{}); + defer file.close(http_io); + try std.testing.expectEqual(read_size, try file.readPositional(http_io, &.{&output}, 0)); + + // A 503 is a server failure there: no hold, one charged retry per + // attempt. + const stats = &http.governor.stats; + try std.testing.expectEqual(@as(u64, 0), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.retries.load(.acquire)); + + try server.stop(io); +} + +test "a server that never lets up fails the request with RateLimited" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const read_size = 256; + + var object: [read_size]u8 = undefined; + @memset(&object, 0x33); + var output: [read_size]u8 = undefined; + + var server = try MockServer.init(io, &object, .{ + .throttle = .{ .always = true, .retry_after_s = 0 }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + var url_buffer: [160]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + const uri: std.Uri = try .parse(url); + const buffers = [_][]u8{&output}; + + var governor: vfs_request.Governor = .init(io, &client, "test", .throttle, .{ + .max_retries = 5, + .initial_delay = .fromMilliseconds(20), + .max_delay = .fromMilliseconds(20), + .max_hold = .fromMilliseconds(50), + .throttle_budget = .fromMilliseconds(300), + }); + const started: std.Io.Timestamp = .now(io, .awake); + // The read never runs: the backend is rate limited past its budget. + // `fileReadPositional` maps this to `Unexpected` with its own log line. + try std.testing.expectError(error.RateLimited, range_read.performRangeRead(&governor, .{ + .request = .{ .target = url }, + }, &uri, range_read.static, &buffers, 0, object.len)); + const elapsed = started.durationTo(.now(io, .awake)).nanoseconds; + try std.testing.expect(elapsed >= 300 * std.time.ns_per_ms); + try std.testing.expect(elapsed < 5 * std.time.ns_per_s); + try std.testing.expect(governor.stats.holds.load(.acquire) > 1); + + try server.stop(io); +} + +test "a reader cancelled while the backend is held returns Canceled" { + const io = std.testing.io; + const read_size = 256; + + var object: [read_size]u8 = undefined; + @memset(&object, 0x21); + var output: [read_size]u8 = undefined; + + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .throttle = .{ .first_gets = 1, .retry_after_s = 30 }, + }, fastRetry(20, 300_000)); + defer fixture.deinit(); + + var failure: std.atomic.Value(u16) = .init(0); + var readers: std.Io.Group = .init; + try readers.concurrent(io, struct { + fn run(file_: std.Io.File, io_: std.Io, out: []u8, failure_: *std.atomic.Value(u16)) void { + const buffers = [_][]u8{out}; + _ = file_.readPositional(io_, &buffers, 0) catch |err| { + failure_.store(@intFromError(err), .release); + return; + }; + } + }.run, .{ fixture.file, fixture.http.io(), output[0..], &failure }); + + // Let the first GET be throttled and the reader settle into the hold. + try io.sleep(.fromMilliseconds(50), .awake); + readers.cancel(io); + try std.testing.expectEqual(@intFromError(error.Canceled), failure.load(.acquire)); + + try fixture.server.stop(io); +} + test "generic HTTP cancels a GET that receives nothing and retries it as a timeout" { const io = std.testing.io; diff --git a/vfs/range_read.zig b/vfs/range_read.zig index 9ab72d91..920720e6 100644 --- a/vfs/range_read.zig +++ b/vfs/range_read.zig @@ -43,9 +43,9 @@ pub fn static(uri: *const std.Uri, _: request.Attempt, _: std.http.Header) anyer /// the governed loop, where `size` is what `data` holds of the `file_size` /// bytes past `offset`; nothing is sent when it is 0. `prepare` is called /// once per attempt: S3's SigV4 signature covers the `Range` header and a -/// timestamp, a bearer token may have expired, and a signed download URL -/// may need re-resolving after a 403. -/// The caller holds its read slot through every retry and sleep; +/// timestamp, a bearer token may have expired during a hold, and a signed +/// download URL may need re-resolving after a 403. +/// The caller holds its read slot through every retry, hold and sleep; /// backend-local parallel readers would multiply the caller's width and /// defeat its memory bounds. pub fn performRangeRead( diff --git a/vfs/request.zig b/vfs/request.zig index 4b006aad..6aa38f24 100644 --- a/vfs/request.zig +++ b/vfs/request.zig @@ -1,8 +1,11 @@ -//! One governed HTTP request loop for every backend: send, classify, and -//! retry a retryable failure after a per-request full-jitter backoff, or -//! after the delay the server names. A backend's open, stat and read -//! failures map to one error contract (`OpenError`, -//! `Governor.readFailure`), each failure logged once. +//! One governed HTTP request loop for every backend: wait out the backend's +//! hold, send, classify. A throttle (429 everywhere, 503 on the object +//! stores) arms a backend-wide hold instead of charging the request's retry +//! budget, so every reader of that backend backs off together and the traffic +//! actually falls; every other retryable failure keeps the per-request +//! backoff. The hold is a deadline, not a permit: nothing is narrowed +//! permanently, recovery is the deadline passing, and a reader's width stays +//! the ceiling it chose, because this loop runs inside its reads. const std = @import("std"); @@ -10,12 +13,19 @@ const log = @import("base.zig").scopedLog(.@"zml/vfs/request"); /// The retry knobs of one remote backend: its `Options.retry`. pub const Retry = struct { - /// Retries per request. + /// Retries per request for failures that are not rate limiting. max_retries: usize = 5, initial_delay: std.Io.Duration = .fromMilliseconds(500), - /// Longest backoff before a request's next attempt, a server-named delay - /// included. + /// Longest backoff before a request's next attempt after a failure that + /// is not a throttle, a server-named delay included. A throttle waits + /// out the backend's hold instead (`max_hold`). max_delay: std.Io.Duration = .fromSeconds(30), + /// Longest hold one throttle may arm over every request of the backend, + /// a server-named delay included. + max_hold: std.Io.Duration = .fromSeconds(120), + /// A throttling episode with no clean window longer than this fails the + /// backend's requests. + throttle_budget: std.Io.Duration = .fromSeconds(300), /// An exchange that receives nothing for this long is canceled and /// retried as a timeout: a peer that stops sending without closing the /// connection would otherwise hold its reader, and the load waiting for @@ -25,8 +35,39 @@ pub const Retry = struct { stall_timeout: std.Io.Duration = .fromSeconds(60), }; -/// One per backend instance: what each of its requests needs and its read -/// statistics. +/// What a hold covers. One governor holds one scope, its backend instance; +/// the authority is carried so that a per-authority map is a local change in +/// `Governor.holdFor`. +pub const Key = []const u8; + +/// The rate-limit scope of a request: its URI authority. Only the host is +/// taken, since `Governor.holdFor` ignores the key; a per-authority scope +/// would want the port beside it. +pub fn authorityOf(uri: std.Uri) Key { + const host = uri.host orelse return ""; + return switch (host) { + .raw => |raw| raw, + .percent_encoded => |encoded| encoded, + }; +} + +/// One scope's rate-limit state, owned by the governor's mutex. +const Hold = struct { + /// When requests may go again; null when nothing is held. + until: ?std.Io.Timestamp = null, + /// The length of the current hold: a clean window that long ends the + /// episode. + length: std.Io.Duration = .fromNanoseconds(0), + /// Holds armed since the episode began; the backoff base. + consecutive: usize = 0, + episode_started: ?std.Io.Timestamp = null, + /// The episode outlived `throttle_budget`; requests fail until a clean + /// window clears it. + exhausted: bool = false, +}; + +/// One per backend instance: what each of its requests needs, its +/// rate-limit state and its read statistics. pub const Governor = struct { pub const Stats = struct { physical_requests: std.atomic.Value(u64) = .init(0), @@ -37,6 +78,10 @@ pub const Governor = struct { server_failures: std.atomic.Value(u64) = .init(0), throttles: std.atomic.Value(u64) = .init(0), retry_delay_ns: std.atomic.Value(u64) = .init(0), + /// Backend-wide holds armed by rate limiting, and the time requests + /// spent waiting them out. + holds: std.atomic.Value(u64) = .init(0), + hold_wait_ns: std.atomic.Value(u64) = .init(0), fn recordAttempt(self: *Stats) void { _ = self.physical_requests.fetchAdd(1, .monotonic); @@ -63,6 +108,16 @@ pub const Governor = struct { const delay_ns: u64 = @intCast(@max(delay.nanoseconds, 0)); _ = self.retry_delay_ns.fetchAdd(delay_ns, .monotonic); } + + /// One rate-limit hold armed over every request of this backend. + fn recordHold(self: *Stats) void { + _ = self.holds.fetchAdd(1, .monotonic); + } + + /// Time one request spent waiting a hold out. + fn recordHoldWait(self: *Stats, waited: std.Io.Duration) void { + _ = self.hold_wait_ns.fetchAdd(@intCast(@max(waited.nanoseconds, 0)), .monotonic); + } }; io: std.Io, @@ -74,22 +129,25 @@ pub const Governor = struct { unavailable: ReadFailure, retry: Retry, stats: Stats = .{}, + mutex: std.Io.Mutex = .init, + hold: Hold = .{}, pub fn init(io: std.Io, client: *std.http.Client, name: []const u8, unavailable: ReadFailure, retry: Retry) Governor { std.debug.assert(retry.initial_delay.nanoseconds >= 0); std.debug.assert(retry.max_delay.nanoseconds >= retry.initial_delay.nanoseconds); + std.debug.assert(retry.max_hold.nanoseconds >= retry.initial_delay.nanoseconds); return .{ .io = io, .client = client, .name = name, .unavailable = unavailable, .retry = retry }; } /// Logs the read totals at debug, once, from the backend's `deinit`; - /// nothing when the backend sent no request. Retries and failures are - /// logged at warn as they happen. + /// nothing when the backend sent no request. Holds, retries and failures + /// are logged at warn as they happen. pub fn logTotals(self: *const Governor) void { const stats = &self.stats; const requests = stats.physical_requests.load(.acquire); if (requests == 0) return; // Under the VFS's scope, `zml/vfs`. - std.log.scoped(.@"zml/vfs").debug("{s} read totals: requests={d}, bytes={Bi:.2}, retries={d}, transient_retries={d}, timeouts={d}, server_failures={d}, throttles={d}, retry_delay_ms={d:.3}", .{ + std.log.scoped(.@"zml/vfs").debug("{s} read totals: requests={d}, bytes={Bi:.2}, retries={d}, transient_retries={d}, timeouts={d}, server_failures={d}, throttles={d}, holds={d}, hold_wait_ms={d:.3}, retry_delay_ms={d:.3}", .{ self.name, requests, stats.physical_bytes.load(.acquire), @@ -98,6 +156,8 @@ pub const Governor = struct { stats.timeouts.load(.acquire), stats.server_failures.load(.acquire), stats.throttles.load(.acquire), + stats.holds.load(.acquire), + @as(f64, @floatFromInt(stats.hold_wait_ns.load(.acquire))) / std.time.ns_per_ms, @as(f64, @floatFromInt(stats.retry_delay_ns.load(.acquire))) / std.time.ns_per_ms, }); } @@ -133,6 +193,15 @@ pub const Governor = struct { }, }; } + + /// The scope `key` belongs to. One generic `HTTP` instance serves every + /// host of its scheme and one `S3` spans buckets, so per-instance + /// over-reaches there; a per-authority scope replaces this body with a + /// map without touching the loop. + fn holdFor(self: *Governor, key: Key) *Hold { + _ = key; + return &self.hold; + } }; /// One attempt as a backend's request hook sees it. @@ -142,7 +211,7 @@ pub const Attempt = struct { governor: *Governor, spec: RequestSpec, /// The status the previous attempt returned, for a hook that reacts to - /// one (an expired signed URL answering 403). + /// one (an expired signed URL answering 403 after a long hold). previous_status: ?std.http.Status = null, /// Bytes received so far, for a hook whose body is long: `exchange` /// cancels an exchange this counter does not move for @@ -157,10 +226,12 @@ const stall_checks = 12; pub const RequestSpec = struct { /// Object identity for log lines. target: []const u8, + /// The rate-limit scope: the request URI's authority. + key: Key = "", /// Statuses this request retries although they are not retryable in /// general, because its hook can do something about them: a signed - /// download URL that expired answers 401 or 403, and the hook - /// re-resolves it. Charged to the retry budget. + /// download URL that expired during a hold answers 401 or 403, and the + /// hook re-resolves it. Charged to the retry budget. retry_once: []const std.http.Status = &.{}, }; @@ -187,11 +258,11 @@ pub fn Outcome(comptime T: type) type { } /// The one request loop. `attemptFn` performs a single HTTP exchange (see -/// `exchange`); this drives the classification and the retry budget around -/// it. Every retryable failure, a throttle and a stall -/// (`Retry.stall_timeout`) included, sleeps the per-request backoff, or the -/// delay the server named up to `max_delay`, and charges one of -/// `max_retries`. A cancel of the caller propagates. +/// `exchange`); this drives the hold, the classification and the two budgets +/// around it. A throttle re-runs the attempt without charging a retry, after +/// the hold it armed; any other retryable failure, a stall +/// (`Retry.stall_timeout`) included, sleeps the per-request backoff and charges +/// one of `max_retries`. A cancel of the caller propagates. pub fn perform( comptime T: type, governor: *Governor, @@ -202,6 +273,7 @@ pub fn perform( var charged: usize = 0; var previous_status: ?std.http.Status = null; while (true) { + try admit(governor, spec); governor.stats.recordAttempt(); var progress: std.atomic.Value(u64) = .init(0); switch (try attemptFn(context, .{ .governor = governor, .spec = spec, .previous_status = previous_status, .progress = &progress })) { @@ -209,6 +281,12 @@ pub fn perform( .retry => |failure| { governor.stats.recordFailure(failure.failure); previous_status = failure.status; + if (failure.failure == .throttle) { + // Not charged to the retry budget: the hold is what + // reduces the traffic, and the episode budget bounds it. + try reportThrottle(governor, spec, failure.delay); + continue; + } if (charged >= governor.retry.max_retries) return error.RetriesExhausted; charged += 1; governor.stats.recordRetry(); @@ -292,6 +370,94 @@ fn watched( return .{ .retry = .{ .failure = .timeout } }; } +/// Waits out the scope's hold. Returns at once when nothing is held; fails +/// with `error.RateLimited` when the episode outlived its budget. Waiters +/// wake at the deadline plus their own jitter, so the readers of one backend +/// do not all fire in the same instant, and re-check, since the hold may have +/// been extended while they slept. +fn admit(governor: *Governor, spec: RequestSpec) error{ Canceled, RateLimited }!void { + while (true) { + const deadline = deadline: { + governor.mutex.lockUncancelable(governor.io); + defer governor.mutex.unlock(governor.io); + const hold = governor.holdFor(spec.key); + const now: std.Io.Timestamp = .now(governor.io, .awake); + refresh(hold, now); + if (hold.exhausted) return error.RateLimited; + const until = hold.until orelse return; + if (until.nanoseconds <= now.nanoseconds) return; + break :deadline until; + }; + const jitter = fullJitterDelay(governor.io, governor.retry.initial_delay, governor.retry.initial_delay, 0); + const started: std.Io.Timestamp = .now(governor.io, .awake); + try std.Io.Clock.Timestamp.wait(deadline.addDuration(jitter).withClock(.awake), governor.io); + governor.stats.recordHoldWait(started.durationTo(.now(governor.io, .awake))); + } +} + +/// Arms or extends the scope's hold after a throttle. `named` is the delay +/// the server asked for, if any; it is honoured up to `max_hold` and never +/// below `initial_delay`, because `Retry-After: 0` and Hugging Face's +/// `RateLimit; t=0` parse to zero, and a throttle charges no retry: a zero +/// hold would be a hot loop. +fn reportThrottle(governor: *Governor, spec: RequestSpec, named: ?std.Io.Duration) error{RateLimited}!void { + const retry = governor.retry; + governor.mutex.lockUncancelable(governor.io); + defer governor.mutex.unlock(governor.io); + const hold = governor.holdFor(spec.key); + const now: std.Io.Timestamp = .now(governor.io, .awake); + refresh(hold, now); + + if (hold.episode_started) |started| { + if (started.durationTo(now).nanoseconds > retry.throttle_budget.nanoseconds) { + hold.exhausted = true; + // The caller reports the failure; this is the one line that + // says why its request never ran. + log.warn("{s}: rate limited for longer than the throttle budget while reading {s}; giving up", .{ governor.name, spec.target }); + return error.RateLimited; + } + } else { + hold.episode_started = now; + } + + const holding = if (hold.until) |until| now.nanoseconds < until.nanoseconds else false; + if (!holding) { + hold.consecutive += 1; + const backoff = fullJitterDelay(governor.io, retry.initial_delay, retry.max_delay, hold.consecutive - 1); + var delay = backoff; + if (named) |value| { + if (value.nanoseconds > delay.nanoseconds) delay = value; + } + hold.length = clampDelay(delay, retry.initial_delay, retry.max_hold); + hold.until = now.addDuration(hold.length); + log.warn("{s}: {s} is rate limiting; holding its requests for {d:.3}s", .{ + governor.name, + spec.target, + @as(f64, @floatFromInt(hold.length.toNanoseconds())) / std.time.ns_per_s, + }); + governor.stats.recordHold(); + return; + } + // A request already in flight when the hold armed got its own answer, + // with a later reset: extend, never shorten. + const asked = named orelse return; + const capped = clampDelay(asked, retry.initial_delay, retry.max_hold); + const extended = now.addDuration(capped); + if (extended.nanoseconds > hold.until.?.nanoseconds) { + hold.until = extended; + hold.length = capped; + } +} + +/// A clean window as long as the hold itself, with no new throttle, ends the +/// episode: nothing else needs counting, since a throttle inside the window +/// would have re-armed `until`. +fn refresh(hold: *Hold, now: std.Io.Timestamp) void { + const until = hold.until orelse return; + if (now.nanoseconds < until.addDuration(hold.length).nanoseconds) return; + hold.* = .{}; +} + fn clampDelay(value: std.Io.Duration, low: std.Io.Duration, high: std.Io.Duration) std.Io.Duration { return .fromNanoseconds(std.math.clamp(value.nanoseconds, low.nanoseconds, high.nanoseconds)); } @@ -518,8 +684,9 @@ pub const HeadChain = struct { /// Follows `start_url` through at most `max_hops` redirects, absolute /// or relative, to the URL that answers a 2xx. Each hop is one governed - /// HEAD, retried as a GET would be. `authorize` names the credentials - /// each URL is sent. An absent or denied object fails the walk + /// HEAD, so a server that rate limits it holds the backend as a + /// throttled GET would. `authorize` names the credentials each URL is + /// sent. An absent or denied object fails the walk /// (`ExchangeOptions.open`). pub fn resolve( self: *HeadChain, @@ -565,7 +732,7 @@ pub const HeadChain = struct { const uri: std.Uri = try .parse(url); const scratch = (current + 1) % self.urls.len; var hop: Hop = .{ .context = context, .url = url, .uri = uri, .location = &self.urls[scratch] }; - const location_len = switch (try perform(Hop.Result, governor, .{ .target = target }, &hop, Hop.attempt)) { + const location_len = switch (try perform(Hop.Result, governor, .{ .target = target, .key = authorityOf(uri) }, &hop, Hop.attempt)) { .end => |size| return .{ .url = url, .size = size }, .redirect => |len| len, }; @@ -653,17 +820,151 @@ fn fullJitterDelay( const test_spec: RequestSpec = .{ .target = "object" }; -fn testGovernor(initial_ms: i64, max_delay_ms: i64) Governor { +fn testGovernor(initial_ms: i64, max_delay_ms: i64, max_hold_ms: i64, budget_ms: i64) Governor { // No attempt in these tests reaches the network: no client. return .init(std.testing.io, undefined, "test", .throttle, .{ .max_retries = 3, .initial_delay = .fromMilliseconds(initial_ms), .max_delay = .fromMilliseconds(max_delay_ms), + .max_hold = .fromMilliseconds(max_hold_ms), + .throttle_budget = .fromMilliseconds(budget_ms), }); } -test "a timeout and a server failure each charge one retry" { - var governor = testGovernor(0, 0); +test "a throttle arms a hold of at least the initial delay, even when the server names zero" { + var governor = testGovernor(20, 20, 1000, 300_000); + + // `Retry-After: 0` and Hugging Face's `RateLimit; t=0` parse to zero. + try reportThrottle(&governor, test_spec, .fromSeconds(0)); + const hold = governor.hold; + try std.testing.expect(hold.until != null); + try std.testing.expectEqual(@as(i96, 20 * std.time.ns_per_ms), hold.length.nanoseconds); + try std.testing.expectEqual(@as(usize, 1), hold.consecutive); + try std.testing.expect(hold.episode_started != null); + try std.testing.expectEqual(@as(u64, 1), governor.stats.holds.load(.acquire)); +} + +test "a longer named delay extends a running hold and a shorter one does not" { + var governor = testGovernor(10, 10, 1000, 300_000); + + try reportThrottle(&governor, test_spec, null); + const armed = governor.hold.until.?; + + // A request that was already in flight got its own answer, with a later + // reset: the hold extends. + try reportThrottle(&governor, test_spec, .fromMilliseconds(500)); + const extended = governor.hold.until.?; + try std.testing.expect(extended.nanoseconds > armed.nanoseconds); + + // A shorter one never shortens it, and neither arms a second hold. + try reportThrottle(&governor, test_spec, .fromMilliseconds(1)); + try std.testing.expectEqual(extended.nanoseconds, governor.hold.until.?.nanoseconds); + try std.testing.expectEqual(@as(usize, 1), governor.hold.consecutive); + try std.testing.expectEqual(@as(u64, 1), governor.stats.holds.load(.acquire)); +} + +test "a named delay is capped by max_hold" { + var governor = testGovernor(10, 10, 50, 300_000); + + try reportThrottle(&governor, test_spec, .fromSeconds(3600)); + try std.testing.expectEqual(@as(i96, 50 * std.time.ns_per_ms), governor.hold.length.nanoseconds); +} + +test "a second throttle inside the clean window doubles the backoff base" { + var governor = testGovernor(10, 1000, 10_000, 300_000); + + try reportThrottle(&governor, test_spec, null); + try std.testing.expectEqual(@as(i96, 10 * std.time.ns_per_ms), governor.hold.length.nanoseconds); + + // The hold has passed but the clean window has not: rewind it by hand + // rather than sleeping through a race. + const now: std.Io.Timestamp = .now(std.testing.io, .awake); + governor.hold.until = now.subDuration(.fromMilliseconds(1)); + try reportThrottle(&governor, test_spec, null); + try std.testing.expectEqual(@as(usize, 2), governor.hold.consecutive); + // Full jitter over twice the base, floored at the initial delay. + try std.testing.expect(governor.hold.length.nanoseconds >= 10 * std.time.ns_per_ms); + try std.testing.expect(governor.hold.length.nanoseconds <= 20 * std.time.ns_per_ms); + try std.testing.expectEqual(@as(u64, 2), governor.stats.holds.load(.acquire)); +} + +test "a clean window as long as the hold ends the episode" { + var governor = testGovernor(10, 1000, 10_000, 300_000); + + try reportThrottle(&governor, test_spec, null); + const length = governor.hold.length; + const now: std.Io.Timestamp = .now(std.testing.io, .awake); + governor.hold.until = now.subDuration(length).subDuration(.fromMilliseconds(1)); + + try reportThrottle(&governor, test_spec, null); + try std.testing.expectEqual(@as(usize, 1), governor.hold.consecutive); + try std.testing.expectEqual(@as(i96, 10 * std.time.ns_per_ms), governor.hold.length.nanoseconds); +} + +test "an episode longer than the throttle budget fails requests until a clean window" { + var governor = testGovernor(10, 10, 10_000, 50); + + try reportThrottle(&governor, test_spec, .fromSeconds(30)); + const now: std.Io.Timestamp = .now(std.testing.io, .awake); + governor.hold.episode_started = now.subDuration(.fromMilliseconds(51)); + + try std.testing.expectError(error.RateLimited, reportThrottle(&governor, test_spec, null)); + try std.testing.expect(governor.hold.exhausted); + // Every other request of this backend fails at once, without waiting the + // hold out. + try std.testing.expectError(error.RateLimited, admit(&governor, test_spec)); + + // The next clean window clears the episode. + governor.hold.until = now.subDuration(governor.hold.length).subDuration(.fromMilliseconds(1)); + try admit(&governor, test_spec); + try std.testing.expect(governor.hold.until == null); + try std.testing.expect(!governor.hold.exhausted); +} + +test "admit returns at once when nothing is held" { + var governor = testGovernor(10, 10, 1000, 300_000); + + const started: std.Io.Timestamp = .now(std.testing.io, .awake); + try admit(&governor, test_spec); + try std.testing.expect(started.durationTo(.now(std.testing.io, .awake)).nanoseconds < 5 * std.time.ns_per_ms); + try std.testing.expectEqual(@as(u64, 0), governor.stats.hold_wait_ns.load(.acquire)); +} + +test "admit waits a hold out and counts the wait" { + var governor = testGovernor(1, 1, 1000, 300_000); + + try reportThrottle(&governor, test_spec, .fromMilliseconds(15)); + const started: std.Io.Timestamp = .now(std.testing.io, .awake); + try admit(&governor, test_spec); + try std.testing.expect(started.durationTo(.now(std.testing.io, .awake)).nanoseconds >= 10 * std.time.ns_per_ms); + try std.testing.expect(governor.stats.hold_wait_ns.load(.acquire) > 0); +} + +test "a waiter blocked on a hold is cancelable" { + const io = std.testing.io; + var governor = testGovernor(1, 1, 100_000, 300_000); + try reportThrottle(&governor, test_spec, .fromSeconds(60)); + + var result: std.atomic.Value(u16) = .init(0); + var started: std.Io.Event = .unset; + var group: std.Io.Group = .init; + try group.concurrent(io, struct { + fn run(governor_: *Governor, started_: *std.Io.Event, result_: *std.atomic.Value(u16)) void { + started_.set(governor_.io); + admit(governor_, test_spec) catch |err| { + result_.store(@intFromError(err), .release); + return; + }; + } + }.run, .{ &governor, &started, &result }); + try started.wait(io); + try io.sleep(.fromMilliseconds(5), .awake); + group.cancel(io); + try std.testing.expectEqual(@intFromError(error.Canceled), result.load(.acquire)); +} + +test "only rate limiting holds: a timeout and a server failure keep the per-request budget" { + var governor = testGovernor(0, 0, 1000, 300_000); const Attempts = struct { seen: usize = 0, @@ -678,12 +979,35 @@ test "a timeout and a server failure each charge one retry" { }; var attempts: Attempts = .{}; try std.testing.expectEqual(@as(usize, 3), try perform(usize, &governor, test_spec, &attempts, Attempts.run)); + try std.testing.expect(governor.hold.until == null); + try std.testing.expectEqual(@as(u64, 0), governor.stats.holds.load(.acquire)); try std.testing.expectEqual(@as(u64, 2), governor.stats.retries.load(.acquire)); try std.testing.expectEqual(@as(u64, 3), governor.stats.physical_requests.load(.acquire)); } -test "the retry budget bounds retryable failures" { - var governor = testGovernor(0, 0); +test "a throttle retries without charging the retry budget" { + var governor = testGovernor(0, 0, 1000, 300_000); + + const Attempts = struct { + seen: usize = 0, + fn run(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + // The retry budget is three; ten throttles do not exhaust it. + if (self.seen <= 10) return .{ .retry = .{ .failure = .throttle, .status = .too_many_requests } }; + try std.testing.expectEqual(@as(?std.http.Status, .too_many_requests), current.previous_status); + return .{ .done = self.seen }; + } + }; + var attempts: Attempts = .{}; + try std.testing.expectEqual(@as(usize, 11), try perform(usize, &governor, test_spec, &attempts, Attempts.run)); + try std.testing.expectEqual(@as(u64, 0), governor.stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 10), governor.stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 11), governor.stats.physical_requests.load(.acquire)); + try std.testing.expect(governor.stats.holds.load(.acquire) >= 1); +} + +test "the retry budget still bounds failures that are not rate limiting" { + var governor = testGovernor(0, 0, 1000, 300_000); const Attempts = struct { seen: usize = 0, @@ -716,7 +1040,7 @@ test "absent and denied statuses map to one error each" { try std.testing.expect(openStatus(status) == null); } - const governor = testGovernor(10, 10); + const governor = testGovernor(10, 10, 1000, 300_000); try std.testing.expect(governor.openFailure("object", error.FileNotFound) == error.FileNotFound); try std.testing.expect(governor.openFailure("object", error.PermissionDenied) == error.PermissionDenied); try std.testing.expect(governor.openFailure("object", error.Canceled) == error.Canceled); @@ -726,7 +1050,7 @@ test "absent and denied statuses map to one error each" { } test "a failed read is Canceled or Unexpected" { - const governor = testGovernor(10, 10); + const governor = testGovernor(10, 10, 1000, 300_000); try std.testing.expect(governor.readFailure("object", 0, error.Canceled) == error.Canceled); try std.testing.expect(governor.readFailure("object", 0, error.RequestFailed) == error.Unexpected); try std.testing.expect(governor.readFailure("object", 0, error.InvalidContentRange) == error.Unexpected); @@ -747,7 +1071,7 @@ test "server retry delay comes from Retry-After seconds or a RateLimit reset" { } test "a server-named retry delay sleeps at most max_delay" { - var governor = testGovernor(1, 20); + var governor = testGovernor(1, 20, 20, 1000); const Failing = struct { seen: usize = 0, fn attempt(self: *@This(), _: Attempt) anyerror!Outcome(usize) { @@ -767,6 +1091,8 @@ test "an attempt that receives nothing is canceled and retried as a timeout" { .max_retries = 3, .initial_delay = .fromMilliseconds(1), .max_delay = .fromMilliseconds(1), + .max_hold = .fromMilliseconds(1), + .throttle_budget = .fromSeconds(1), .stall_timeout = .fromMilliseconds(120), }); const Hang = struct { @@ -804,6 +1130,8 @@ test "an attempt waiting on a request of its own is not a stall" { .max_retries = 3, .initial_delay = .fromMilliseconds(1), .max_delay = .fromMilliseconds(1), + .max_hold = .fromMilliseconds(1), + .throttle_budget = .fromSeconds(1), .stall_timeout = .fromMilliseconds(120), }); const Nested = struct { @@ -860,3 +1188,8 @@ test "an exchange canceled while it waits for the head returns Canceled, not a r try server.stop(io); } + +test "the request authority is the rate-limit scope" { + const uri: std.Uri = try .parse("https://huggingface.co/api/models/x"); + try std.testing.expectEqualStrings("huggingface.co", authorityOf(uri)); +} diff --git a/vfs/vfs.zig b/vfs/vfs.zig index 557eaad2..66604722 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -16,12 +16,14 @@ //! allocate. Any other failure of a request is logged at err with the //! backend and the path, and surfaces as `Unexpected`. A positional read //! that still fails after its retries is logged at err and surfaces as -//! `Unexpected`; a streaming read as `InputOutput`. A remote path is -//! read-only: creating or deleting a file or a directory there, or opening -//! a file for writing, is logged at err and fails with `ReadOnlyFileSystem`. -//! A local file fails as the inner io does (a denied one is `AccessDenied` -//! there); it may be read past the page cache (`useDirectIo`). A backend on -//! `request.zig` logs its read totals at debug from its `deinit`. +//! `Unexpected`; a streaming read as `InputOutput`. A throttled backend +//! holds its requests rather than failing them, until its throttle budget +//! runs out (`request.Retry`). A remote path is read-only: creating or +//! deleting a file or a directory there, or opening a file for writing, is +//! logged at err and fails with `ReadOnlyFileSystem`. A local file fails as +//! the inner io does (a denied one is `AccessDenied` there); it may be read +//! past the page cache (`useDirectIo`). A backend on `request.zig` logs its +//! read totals at debug from its `deinit`. const std = @import("std"); const stdx = @import("stdx"); From 153b9e3fad5873187269dd275703e41daf0064fa Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:33:40 +0100 Subject: [PATCH 039/103] zml/io: add ShardGrid ShardGrid is the first file of the new loader under zml/io/, which zml does not export yet: io.zig's test block is the only thing that reaches it. It places a destination's logical bytes on its devices by arithmetic, never by listing rows: init takes the shape as PJRT allocates it, placed on the platform and with sub-byte storage packed, and computes the even box grid; locate maps a logical byte to its box and the offset inside it, rankIn counts a box's bytes before a logical offset, runEnd and runCount cut a range into the runs that stay in one box, and replicas gives the devices per box. Platform.placedShape names the default a buffer already applies to a shape without a mesh: replicated over every device. The tests walk every fixed-step byte range of replicated, packed sub-byte, 1D mirrored and folded, 2D, 3D and scalar placements on 4- and 8-device CPU platforms and compare each byte with Placement.slices. They skip where no CPU plugin is available. Co-Authored-By: Claude Opus 5.5 --- zml/BUILD.bazel | 1 + zml/io.zig | 6 + zml/io/ShardGrid.zig | 352 +++++++++++++++++++++++++++++++++++++++++++ zml/platform.zig | 8 + 4 files changed, 367 insertions(+) create mode 100644 zml/io/ShardGrid.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 8188c743..538524f8 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -86,6 +86,7 @@ zig_library( "exe.zig", "floats.zig", "io.zig", + "io/ShardGrid.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/moe/moe.zig", diff --git a/zml/io.zig b/zml/io.zig index ae41a5b7..6754a377 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -24,6 +24,12 @@ const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/io"); +// The new loader's files under io/, which zml does not export: only their +// tests reach them, from here. +test { + _ = @import("io/ShardGrid.zig"); +} + pub const TensorStore = struct { pub const Binding = struct { tensors: []*safetensors.Tensor, diff --git a/zml/io/ShardGrid.zig b/zml/io/ShardGrid.zig new file mode 100644 index 00000000..41969285 --- /dev/null +++ b/zml/io/ShardGrid.zig @@ -0,0 +1,352 @@ +//! Where a destination's logical bytes land. Its devices hold the boxes of an +//! even grid over the tensor (`Sharding.Placement` splits every axis into +//! equal shards), each box packed row-major on its devices, and devices +//! holding the same box are one writer set. Box order and logical order agree +//! on a box's elements, so a contiguous logical range lands as one contiguous +//! range of every box it touches: ranges are placed by arithmetic, never by +//! listing rows. +//! +//! For a [4, 8] tensor whose columns are split in two, each letter one +//! element: +//! +//! Logical bytes Box 0 Box 1 +//! a b c d e f g h a b c d e f g h +//! i j k l m n o p i j k l m n o p +//! q r s t u v w x q r s t u v w x +//! y z A B C D E F y z A B C D E F +//! +//! The logical range from c to n lands as c..l on box 0 and e..n on box 1. +//! Runs are the aligned pieces that stay in one box (`run`: a shard of the +//! innermost split axis, four elements here); a worker walks them to place +//! a range's bytes, the planner only counts them. + +const std = @import("std"); + +const constants = @import("../constants.zig"); +const Platform = @import("../platform.zig").Platform; +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); + +const ShardGrid = @This(); + +const max_rank = constants.MAX_RANK; +/// Every box has at least one device, and a writer mask holds 64. +pub const max_boxes = 64; + +rank: usize, +/// Elements per axis, of the tensor and of one box. +dims: [max_rank]u64 = @splat(1), +shard: [max_rank]u64 = @splat(1), +/// Byte strides, logical and inside a box. +strides: [max_rank]u64 = @splat(0), +box_strides: [max_rank]u64 = @splat(0), +/// A box's index is row-major over the shard counts of the axes. +index_strides: [max_rank]u64 = @splat(0), +element: u64, +total: u64, +box_bytes: u64, +/// Aligned runs of `run` logical bytes each land in one box, contiguous. +run: u64, +box_count: usize, +/// The canonical device indices holding each box. +writers: [max_boxes]u64 = @splat(0), +/// One box: the shape each device holds. +local: Shape, +/// The devices in canonical order, which the `writers` masks index; +/// borrowed from the placed shape's mesh (the platform's when `shape` names +/// none). +devices: []const Sharding.Device, + +/// The grid of `shape` placed on `platform` (`Platform.placedShape`), its +/// sub-byte storage packed (`Shape.packedShape`): what PJRT allocates. +/// `shape` must be non-empty and place on the platform's devices, as the +/// loader's front end checked. +pub fn init(platform: *const Platform, shape: Shape) ShardGrid { + const placed = platform.placedShape(shape).packedShape(); + std.debug.assert(placed.byteSize() != 0); + const placement = Sharding.Placement.init(&placed) catch |err| + std.debug.panic("ShardGrid: {f} does not place, though checked: {s}", .{ placed, @errorName(err) }); + const local = placement.localShape(placed); + const devices = placement.mesh().?.devicesInCanonicalOrder(); + std.debug.assert(devices.len <= max_boxes); + const element: u64 = placed.dtype().sizeOf(); + var grid: ShardGrid = .{ + .rank = placed.rank(), + .element = element, + .total = placed.byteSize(), + .box_bytes = local.byteSize(), + .run = placed.byteSize(), + .box_count = 1, + .local = local, + .devices = devices, + }; + const strides = placed.computeByteStrides(); + var box_stride = element; + var index_stride: u64 = 1; + var axis = grid.rank; + while (axis > 0) { + axis -= 1; + const dim: u64 = @intCast(placed.dim(axis)); + const shard: u64 = @intCast(local.dim(axis)); + grid.dims[axis] = dim; + grid.shard[axis] = shard; + grid.strides[axis] = @intCast(strides.get(axis)); + grid.box_strides[axis] = box_stride; + grid.index_strides[axis] = index_stride; + box_stride *= shard; + const count = dim / shard; + index_stride *= count; + // The innermost split axis cuts the runs. + if (count > 1 and grid.run == grid.total) grid.run = shard * grid.strides[axis]; + } + // A placement holds every box on at least one of its devices. + std.debug.assert(index_stride <= devices.len); + grid.box_count = @intCast(index_stride); + for (0..devices.len) |writer| { + const slices = placement.slices(writer, placed); + var box: u64 = 0; + for (slices.constSlice(), 0..) |slice, i| box += @as(u64, @intCast(slice.start)) / grid.shard[i] * grid.index_strides[i]; + grid.writers[@intCast(box)] |= @as(u64, 1) << @intCast(writer); + } + for (grid.writers[0..grid.box_count]) |mask| std.debug.assert(mask != 0); + return grid; +} + +fn coordinate(self: *const ShardGrid, offset: u64, axis: usize) u64 { + return offset / self.strides[axis] % self.dims[axis]; +} + +/// The box holding logical byte `offset`, and where the byte lands inside it. +pub fn locate(self: *const ShardGrid, offset: u64) struct { usize, u64 } { + var box: u64 = 0; + var at = offset % self.element; + for (0..self.rank) |i| { + const c = self.coordinate(offset, i); + box += c / self.shard[i] * self.index_strides[i]; + at += c % self.shard[i] * self.box_strides[i]; + } + return .{ @intCast(box), at }; +} + +/// The bytes of box `box` before logical offset `offset`: a logical range +/// `[start, end)` lands on that box at `[rankIn(start), rankIn(end))`. +/// Counted axis by axis, as in a lexicographic comparison. +pub fn rankIn(self: *const ShardGrid, box: usize, offset: u64) u64 { + if (offset >= self.total) return self.box_bytes; + var before: u64 = 0; + for (0..self.rank) |i| { + const c = self.coordinate(offset, i); + const first = @as(u64, box) / self.index_strides[i] % (self.dims[i] / self.shard[i]) * self.shard[i]; + before += @min(c -| first, self.shard[i]) * self.box_strides[i]; + if (c < first or c >= first + self.shard[i]) return before; + } + return before + offset % self.element; +} + +/// The end of the run holding logical byte `offset`, at most `limit`. +pub fn runEnd(self: *const ShardGrid, offset: u64, limit: u64) u64 { + return @min(limit, (offset / self.run + 1) * self.run); +} + +/// The runs with a byte in `[start, end)`, `start < end`. +pub fn runCount(self: *const ShardGrid, start: u64, end: u64) u64 { + return (end + self.run - 1) / self.run - start / self.run; +} + +/// Devices per box: the grid is even, so every box has as many. +pub fn replicas(self: *const ShardGrid) usize { + return @popCount(self.writers[0]); +} + +const testing = struct { + const CreateOptions = @import("../platform.zig").CreateOptions; + + const Scenario = struct { + name: []const u8, + device_count: u32, + physical_mesh: CreateOptions.PhysicalMesh = .auto, + shape: Shape, + logical_mesh: Sharding.LogicalMesh, + strategy: Sharding.Strategy, + /// Logical ranges are cut every `step` bytes, unaligned to rows. + step: u64, + }; + + fn run(scenario: Scenario, partitioning: anytype) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + var platform = Platform.init(allocator, io, .cpu, .{ + .physical_mesh = scenario.physical_mesh, + .cpu = .{ .device_count = scenario.device_count }, + }) catch return error.SkipZigTest; + defer platform.deinit(allocator, io); + const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + try expectPlaces(allocator, platform, scenario.shape.withPartitioning(&mesh, partitioning), scenario.step); + } + + /// Walks the runs of every `step`-byte range into the writers' boxes and + /// compares with each device's slices, element by element; checks that + /// `rankIn` bounds the range's bytes on each box and that `runCount` + /// counts the runs. + fn expectPlaces(allocator: std.mem.Allocator, platform: *const Platform, shape: Shape, step: u64) !void { + const grid: ShardGrid = .init(platform, shape); + const placed = platform.placedShape(shape).packedShape(); + const placement: Sharding.Placement = try .init(&placed); + const box_bytes: usize = @intCast(grid.box_bytes); + const total: usize = @intCast(grid.total); + const source = try allocator.alloc(u8, total); + defer allocator.free(source); + for (source, 0..) |*byte, i| byte.* = @truncate(i *% 131 +% 17); + + const expected = try allocator.alloc(u8, grid.devices.len * box_bytes); + defer allocator.free(expected); + const element: usize = @intCast(grid.element); + for (0..grid.devices.len) |writer| { + const slices = placement.slices(writer, placed); + var index: [max_rank]u64 = @splat(0); + var cursor: usize = 0; + while (cursor < box_bytes) : (cursor += element) { + var logical: u64 = 0; + for (0..grid.rank) |i| logical += (@as(u64, @intCast(slices.get(i).start)) + index[i]) * grid.strides[i]; + @memcpy(expected[writer * box_bytes + cursor ..][0..element], source[@intCast(logical)..][0..element]); + var axis = grid.rank; + while (axis > 0) { + axis -= 1; + index[axis] += 1; + if (index[axis] < grid.shard[axis]) break; + index[axis] = 0; + } + } + } + + const actual = try allocator.alloc(u8, expected.len); + defer allocator.free(actual); + @memset(actual, 0); + var start: u64 = 0; + while (start < grid.total) : (start += step) { + const end = @min(grid.total, start + step); + var low: [max_boxes]u64 = @splat(std.math.maxInt(u64)); + var high: [max_boxes]u64 = @splat(0); + var runs: u64 = 0; + var at = start; + while (at < end) : (runs += 1) { + const stop = grid.runEnd(at, end); + const box, const landing = grid.locate(at); + low[box] = @min(low[box], landing); + high[box] = @max(high[box], landing + stop - at); + var mask = grid.writers[box]; + while (mask != 0) : (mask &= mask - 1) { + const writer: usize = @ctz(mask); + @memcpy(actual[writer * box_bytes + @as(usize, @intCast(landing)) ..][0..@intCast(stop - at)], source[@intCast(at)..@intCast(stop)]); + } + at = stop; + } + try std.testing.expectEqual(runs, grid.runCount(start, end)); + for (0..grid.box_count) |box| { + const first = grid.rankIn(box, start); + const last = grid.rankIn(box, end); + if (high[box] == 0) { + try std.testing.expectEqual(first, last); + } else { + try std.testing.expectEqual(low[box], first); + try std.testing.expectEqual(high[box], last); + } + } + } + try std.testing.expectEqualSlices(u8, expected, actual); + } +}; + +test "a grid places replicated tensors, across range boundaries" { + try testing.run(.{ + .name = "replicated", + .device_count = 4, + .shape = Shape.init(.{ .rows = 9, .cols = 257 }, .f32), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + .step = 773, + }, .{ .rows = .replicated, .cols = .replicated }); +} + +test "a grid places packed sub-byte storage" { + const logical = Shape.init(.{ .rows = 9, .cols = 256 }, .u2); + try testing.run(.{ + .name = "packed_u2", + .device_count = 4, + .shape = logical, + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + .step = 131, + }, .{ .rows = .replicated, .cols = .x }); +} + +test "a grid places 1D mirrored and folded sharding" { + try testing.run(.{ + .name = "mirrored_1d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .rows = 7, .model = 1024 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .step = 2053, + }, .{ .rows = .replicated, .model = .model }); + try testing.run(.{ + .name = "folded_1d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); + break :blk strategy; + }, + .step = 3001, + }, .{ .model = .model }); +} + +test "a grid places 2D and 3D sharding" { + try testing.run(.{ + .name = "batch_model_2d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), + .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), + .step = 4093, + }, .{ .batch = .batch, .model = .model }); + try testing.run(.{ + .name = "folded_model_3d", + .device_count = 8, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + break :blk strategy; + }, + .step = 8191, + }, .{ .batch = .replicated, .model = .model }); + // Three axes, the middle and the last split: runs are a quarter row. + try testing.run(.{ + .name = "middle_and_last_3d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .a = 3, .b = 4, .c = 8 }, .u16), + .logical_mesh = .mesh(.{ .x = .low_bandwidth, .y = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x, .y = .link_y }), + .step = 37, + }, .{ .a = .replicated, .b = .x, .c = .y }); +} + +test "a grid places a scalar on every device" { + try testing.run(.{ + .name = "scalar", + .device_count = 4, + .shape = Shape.init(.{}, .f32), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + .step = 3, + }, .{}); +} diff --git a/zml/platform.zig b/zml/platform.zig index 25f809ed..f2401715 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -606,6 +606,14 @@ pub const Platform = struct { try writer.writeAll(" }"); } + /// `shape` placed as a buffer places it: on its mesh, or replicated on + /// every device (`replicated_mesh`) when it names none. + pub fn placedShape(self: *const Platform, shape: zml.Shape) zml.Shape { + var placed = shape; + if (placed._sharding.mesh == null) placed._sharding.mesh = self.replicated_mesh; + return placed; + } + pub fn memoryKind(self: *const Platform, kind: Memory.Kind) []const u8 { for (self.memories) |mem| { if (mem.isOfKind(kind)) { From e74d3d23223f570faed16c772dfa7ebaec7a0f61 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:34:51 +0100 Subject: [PATCH 040/103] zml/io: add the owning TensorStore core beside the old store zml/io/TensorStore.zig is the new loader's store. zml does not export it: zml.io.TensorStore in io.zig stays the store models and the current loader use, and keeps borrowing its registry. The new store takes the registry's entries instead: init leaves the caller's registry empty, so its deferred deinit stays correct, and locks the map's pointers until deinit, so an entry's address identifies its checkpoint tensor and a later insert panics in safe builds. It has the old store's caller API: init with Options{meshes, quantization}, deinit, view and mesh, and View's root, withPrefix, withLayer, prefix, get, count, mesh and quantization. Memory names where the loader delivers a tensor (default, host_pinned or host_mapped), with isHost and the memory space of each device's buffer. zml/io/log.zig adds the new loader's log scopes, load, io and mem. Their err logs at debug in test builds, where the test runner counts a logged error as a failure, and failure appends the error's name and logs cancellation and OutOfMemory at debug. Co-Authored-By: Claude Opus 5.5 --- zml/BUILD.bazel | 2 + zml/io.zig | 2 + zml/io/TensorStore.zig | 222 +++++++++++++++++++++++++++++++++++++++++ zml/io/log.zig | 38 +++++++ 4 files changed, 264 insertions(+) create mode 100644 zml/io/TensorStore.zig create mode 100644 zml/io/log.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 538524f8..cee65d79 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -87,6 +87,8 @@ zig_library( "floats.zig", "io.zig", "io/ShardGrid.zig", + "io/TensorStore.zig", + "io/log.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/moe/moe.zig", diff --git a/zml/io.zig b/zml/io.zig index 6754a377..8afb9636 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -28,6 +28,8 @@ const log = std.log.scoped(.@"zml/io"); // tests reach them, from here. test { _ = @import("io/ShardGrid.zig"); + _ = @import("io/TensorStore.zig"); + _ = @import("io/log.zig"); } pub const TensorStore = struct { diff --git a/zml/io/TensorStore.zig b/zml/io/TensorStore.zig new file mode 100644 index 00000000..969ed704 --- /dev/null +++ b/zml/io/TensorStore.zig @@ -0,0 +1,222 @@ +//! The new loader's store: prefixed views over the entries it takes from a +//! checkpoint registry. zml does not export it: models and the loader they +//! use build `zml.io.TensorStore` (io.zig), which borrows its registry, so +//! production changes go there and the new loader's here. +//! +//! The store owns those entries. Views point to it: once `view` was called +//! it must not move, and it must outlive them. It borrows the meshes of +//! `Options.meshes`, which belong to the platform. +const std = @import("std"); + +const stdx = @import("stdx"); + +const MemorySpace = @import("../platform.zig").Memory.Kind; +const Quantization = @import("../quantization.zig").Quantization; +const safetensors = @import("../safetensors.zig"); +const Sharding = @import("../Sharding.zig"); + +const TensorStore = @This(); + +/// Where the loader delivers a loaded tensor, on every device of its mesh. +/// The destination is allocated once per device; the host kinds take no +/// device memory. +pub const Memory = enum { + /// Device memory. + default, + /// Buffers in each device's pinned host memory space. An executable + /// that takes them as device inputs copies them to the device first. + host_pinned, + /// Pinned host memory that executables read in place: each device's + /// buffer is a device-kind view of it (`createViewOfDeviceBuffer`), + /// and the memory goes with the last view. For a table too large for + /// device memory, of which a step reads a few rows. XLA counts the + /// views against device memory. + host_mapped, + + /// Whether the bytes live in host memory, written by the CPU. + pub fn isHost(self: Memory) bool { + return switch (self) { + .default => false, + .host_pinned, .host_mapped => true, + }; + } + + /// The memory space of each device's delivered buffer. + pub fn space(self: Memory) MemorySpace { + return switch (self) { + .default, .host_mapped => .default, + .host_pinned => .host_pinned, + }; + } +}; + +registry: safetensors.TensorRegistry, +meshes: []const *const Sharding.Mesh, +quantization: ?Quantization.Target, +arena: std.heap.ArenaAllocator, + +pub const Options = struct { + /// The meshes `View.mesh` names. The slice is copied; the meshes must + /// outlive the store. + meshes: []const *const Sharding.Mesh = &.{}, + /// How the platform's FP8 GEMMs want block-FP8 weights, for the code + /// that binds them (`View.quantization`): `.of(platform)`. Required to + /// bind block-FP8 weights. + quantization: ?Quantization.Target = null, +}; + +/// A store over the entries of `registry`, which must be complete: add +/// every checkpoint first (`TensorRegistry.addRepo`). It takes the entries +/// and leaves `registry` empty. The store's map stays locked until +/// `deinit`, so a later insert panics in safe builds, and an entry's +/// address is its checkpoint tensor's identity (`View.get`). +pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, opts: Options) TensorStore { + var arena: std.heap.ArenaAllocator = .init(allocator); + const meshes = arena.allocator().dupe(*const Sharding.Mesh, opts.meshes) catch @panic("OOM"); + var self: TensorStore = .{ + .registry = registry.*, + .meshes = meshes, + .quantization = opts.quantization, + .arena = arena, + }; + registry.* = .init(registry.arena.child_allocator); + self.registry.tensors.lockPointers(); + return self; +} + +/// Frees the registry entries `init` took. +pub fn deinit(self: *TensorStore) void { + self.arena.deinit(); + self.registry.tensors.unlockPointers(); + self.registry.deinit(); +} + +/// The root view, with no prefix. It points to this store, which must not +/// move afterwards. +pub fn view(self: *TensorStore) View { + return .{ .store = self }; +} + +/// The mesh `name` among `Options.meshes`; model code asks its `View`. +pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (store.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + std.debug.panic( + \\Found no meshes named {s} in TensorStore. + \\Try passing more meshes to `zml.io.TensorStore.init`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(store.meshes) }); +} + +/// What model code holds: a key prefix over the store's checkpoint, to look +/// tensors up. A value that keeps its own copy of the prefix (256 bytes at +/// most, longer panics) and points to the store. +pub const View = struct { + store: *TensorStore, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + /// The store's root view, with no prefix. + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + /// This view under `prefix_` and a dot: `withPrefix("model")` then + /// `get("embed")` looks up `model.embed`. + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix(), prefix_ }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + /// This view under `index` and a dot, as `withPrefix` of the number. + pub fn withLayer(self: *const View, index: usize) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix(), index }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + /// The keys' prefix under this view, with its trailing dot; "" at the + /// root. + pub fn prefix(self: *const View) []const u8 { + return self.prefix_buffer[0..self.prefix_length]; + } + + /// The checkpoint tensor `subkey`, the registry's own entry, whose + /// address is the tensor's identity. Null when the checkpoint has none. + pub fn get(self: View, subkey: []const u8) ?*const safetensors.Tensor { + var buffer: [256]u8 = undefined; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); + return self.store.registry.tensors.getPtr(key); + } + + /// See `TensorStore.mesh`. + pub fn mesh(self: View, name: @EnumLiteral()) *const Sharding.Mesh { + return self.store.mesh(name); + } + + /// See `Options.quantization`. Panics when the store has none: binding + /// block-FP8 weights requires it. + pub fn quantization(self: View) Quantization.Target { + return self.store.quantization orelse + stdx.debug.panic("TensorStore.Options.quantization is unset: block-FP8 weights bind for a platform; pass .of(platform)", .{}); + } + + /// The checkpoint tensors under this view's prefix: a scan of every key. + pub fn count(self: View) usize { + var count_: usize = 0; + const prefix_ = self.prefix(); + var it = self.store.registry.tensors.iterator(); + while (it.next()) |item| { + const key = item.key_ptr.*; + if (std.mem.startsWith(u8, key, prefix_)) { + count_ += 1; + } + } + return count_; + } + + fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { + const key = std.fmt.bufPrint(buffer, fmt, args) catch + std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); + return key; + } +}; + +test "get finds exact keys, the registry's own entries, and count the keys under a prefix" { + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "layer.experts.0.down_proj.weight", .shape = .init(.{4}, .u8), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + // The store took the entries. + try std.testing.expectEqual(0, registry.tensors.count()); + + const layer = store.view().withPrefix("layer"); + const entry = layer.get("experts.0.down_proj.weight").?; + try std.testing.expectEqual(entry, store.registry.tensors.getPtr("layer.experts.0.down_proj.weight").?); + try std.testing.expectEqual(null, layer.get("experts.0.down_proj")); + try std.testing.expectEqual(null, layer.get("weights")); + try std.testing.expectEqualStrings("layer.", layer.prefix()); + try std.testing.expectEqualStrings("", layer.root().prefix()); + try std.testing.expectEqualStrings("layer.experts.0.", layer.withPrefix("experts").withLayer(0).prefix()); + try std.testing.expectEqual(1, layer.withPrefix("experts").count()); + try std.testing.expectEqual(0, layer.withPrefix("gate").count()); +} diff --git a/zml/io/log.zig b/zml/io/log.zig new file mode 100644 index 00000000..e5bb5820 --- /dev/null +++ b/zml/io/log.zig @@ -0,0 +1,38 @@ +//! The loader's log scopes. A failure is logged once, where it is +//! detected, at `err` with the values that explain it; cancellation and a +//! plain allocation failure log at `debug`. Frames that only propagate an +//! error do not log it again. Test builds log `err` at `debug`, since the +//! test runner counts logged errors as failures: error-path tests assert +//! the returned errors and the cleanup, not the logs. +const std = @import("std"); +const builtin = @import("builtin"); + +pub const load = Scoped(.@"zml/io/load"); +pub const io = Scoped(.@"zml/io"); +pub const mem = Scoped(.@"zml/mem"); + +fn Scoped(comptime scope: @EnumLiteral()) type { + const base = std.log.scoped(scope); + return struct { + pub const debug = base.debug; + pub const info = base.info; + pub const warn = base.warn; + + /// A terminal failure. The test runner counts logged errors as + /// test failures, so test builds log it at `debug` instead. + pub fn err(comptime fmt: []const u8, args: anytype) void { + if (builtin.is_test) base.debug(fmt, args) else base.err(fmt, args); + } + + /// The first failure of a pipeline, with the error's name appended. + /// Cancellation is expected, and an `OutOfMemory` is a plain + /// allocation failure, returned unlogged, or was logged where it + /// failed (host memory): those log at `debug`. + pub fn failure(e: anyerror, comptime fmt: []const u8, args: anytype) void { + const named_fmt = fmt ++ ": {s}"; + const named_args = args ++ .{@errorName(e)}; + const logged = e == error.Canceled or e == error.Cancelled or e == error.OutOfMemory; + if (logged) debug(named_fmt, named_args) else err(named_fmt, named_args); + } + }; +} From 88514c800f78a5b9e4babbd1341a61c03b8e0fa8 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:38:08 +0100 Subject: [PATCH 041/103] zml/buffer: walk the shape's mesh and add testing.Devices and expectShards Buffer.from, Buffer.uninitialized and HostAccessible.init created one shard for every device of the platform, and toSlice and toSliceAlloc read as many, whatever mesh the shape was placed on. A buffer on a mesh over fewer devices then got shards on the wrong devices, or read past its shards. They now walk the placement's mesh in its canonical device order, and the readbacks walk the buffer's own shards. buffer.deviceMemory finds a mesh device's memory of a kind, and panics naming the device when it has none, where only device 0 was checked. placementOrPanic places a shape without a mesh through Platform.placedShape. On a mesh over every device nothing changes. The test puts a buffer on two of four devices, out of the platform's order, in default and pinned host memory. It needs test infrastructure: - testing.Devices gives N devices, the accelerator's when it has as many, else a CPU platform of its own, with a one-axis physical mesh over exactly them; - testing.expectShards reads every shard back in mesh order and checks its memory space and its box of the tensor's row-major bytes, for a delivery kind of the new loader's store; - Buffer.Shard.toHost copies one shard to the host and waits. Co-Authored-By: Claude Opus 5.5 --- zml/buffer.zig | 78 +++++++++++++++++++++++------ zml/buffer/HostAccessible.zig | 5 +- zml/testing.zig | 93 +++++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 16 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 9da62c68..45bc2be6 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -55,6 +55,16 @@ pub const Buffer = struct { pub fn devicePtr(self: *const Shard) *anyopaque { return self._pjrt_buffer.opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; } + + /// Copies this shard's bytes to `destination`, which must hold the + /// shard's byte size, and waits for the copy. + pub fn toHost(self: *const Shard, io: std.Io, destination: []u8) !void { + const maybe_event = try self._pjrt_buffer.toHostBuffer(self._platform.pjrt_api, destination); + if (maybe_event) |event| { + defer event.deinit(self._platform.pjrt_api); + try event.await(self._platform.pjrt_api, io); + } + } }; pub const ShardIterator = struct { @@ -118,6 +128,8 @@ pub const Buffer = struct { return @intCast(self._shards.len); } + /// One shard per device of the shape's mesh (the platform's replicated + /// mesh when it names none), in that mesh's canonical device order. pub fn shards(self: *const Buffer) ShardIterator { return .{ ._platform = self._platform, @@ -155,7 +167,6 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); const byte_size: usize = if (packed_shape.rank() == 0) packed_shape.dtype().sizeOf() else @@ -163,8 +174,8 @@ pub const Buffer = struct { stdx.debug.assert(data_.len == byte_size, "Expected data length {}, got {}", .{ byte_size, data_.len }); const base_ptr = data_.ptr; - for (platform.physical_mesh.devices_in_canonical_order, 0..) |device, device_index| { - const memory = platform.devices[device.id].memory(opts.memory).?; + for (placement.mesh().?.devicesInCanonicalOrder(), 0..) |device, device_index| { + const memory = deviceMemory(platform, device, opts.memory); const args: pjrt.Client.BufferFromHostBufferArgs = .{ // Change for each device .data = placement.shardPtr(device_index, base_ptr), @@ -247,10 +258,8 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); - - for (platform.physical_mesh.devices_in_canonical_order) |device| { - const memory = platform.devices[device.id].memory(opts.memory).?; + for (metadata.placement.mesh().?.devicesInCanonicalOrder()) |device| { + const memory = deviceMemory(platform, device, opts.memory); const args: pjrt.Client.CreateUninitializedBufferArgs = .{ // Change for each device .dst = .{ .memory = memory.pjrt_memory }, @@ -294,14 +303,14 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); const placement = placementOrPanic(self._platform, self._shape); - for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + for (self._shards.constSlice(), 0..) |shard, shard_index| { // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed const sub_slice = placement.shardSlice(shard_index, slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; const size_bytes = placement.localShape(self._shape).byteSize(); const destination = sub_slice.data()[0..size_bytes]; - const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); + const maybe_event = try shard.toHostBuffer(self._platform.pjrt_api, destination); if (maybe_event) |event| { defer event.deinit(self._platform.pjrt_api); @@ -321,9 +330,9 @@ pub const Buffer = struct { var shard_slice = try Slice.alloc(allocator, placement.localShape(self._shape)); defer shard_slice.free(allocator); - for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + for (self._shards.constSlice(), 0..) |shard, shard_index| { const sub_slice = placement.shardSlice(shard_index, slice); - const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); + const maybe_event = try shard.toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { defer event.deinit(self._platform.pjrt_api); try event.await(self._platform.pjrt_api, io); @@ -420,6 +429,41 @@ test "device round-trip" { } } +test "a buffer on a sub-mesh has one shard per device of that mesh" { + const io = std.testing.io; + const allocator = std.testing.allocator; + // Four devices: the accelerator's when it has them, else a CPU platform's + // (none on a host without the CPU plugin, such as aarch64 Linux). + var devices: testing.Devices = undefined; + devices.init(allocator, io, 4) catch return error.SkipZigTest; + defer devices.deinit(); + const platform = devices.platform; + // Two of the four devices, out of the platform's order. + const nodes = [_]Sharding.PhysicalNode{ .device(platform.devices[3]), .device(platform.devices[1]) }; + var physical: Sharding.PhysicalMesh = try .fromTree(allocator, platform.target, .axis(.bus, .tree, &nodes)); + defer physical.deinit(allocator); + const mesh: Sharding.Mesh = try .init("pair", &physical, .mesh(.{ .model = .high_bandwidth }), .parseBindings(.{ .model = .bus })); + + var values: [8]u32 = undefined; + for (&values, 0..) |*value, i| value.* = @intCast(i); + const shape = Shape.init(.{ .b = 4, .d = 2 }, .u32).withPartitioning(&mesh, .{ .b = .model }); + var buffer: Buffer = try .fromBytes(io, platform, shape, std.mem.sliceAsBytes(&values)); + defer buffer.deinit(); + try testing.expectShards(allocator, io, &buffer, std.mem.sliceAsBytes(&values), .default); + + const read: Slice = try buffer.toSliceAlloc(allocator, io); + defer read.free(allocator); + try std.testing.expectEqualSlices(u32, &values, read.constItems(u32)); + var into: [8]u32 = undefined; + try buffer.toSlice(io, .init(shape, std.mem.sliceAsBytes(&into))); + try std.testing.expectEqualSlices(u32, &values, &into); + + const pinned: Buffer.HostAccessible = try .init(allocator, io, platform, shape, .undonatable); + defer pinned.deinit(allocator); + pinned.view(u32).copyFrom(&values); + try testing.expectShards(allocator, io, &pinned.buffer, std.mem.sliceAsBytes(&values), .host_pinned); +} + pub const PjrtBufferMetadata = struct { shape: Shape, ty: pjrt.BufferType, @@ -445,10 +489,16 @@ pub const PjrtBufferMetadata = struct { } }; +/// The `kind` memory of `device`, a device of a buffer's mesh, which names it +/// by its index in `platform.devices`. Outside `Buffer` for `Buffer.HostAccessible`. +pub fn deviceMemory(platform: *const Platform, device: Sharding.Device, kind: Memory.Kind) *const Memory { + stdx.debug.assert(device.id < platform.devices.len, "Mesh device {d} is not one of the {d} devices of {t}", .{ device.id, platform.devices.len, platform.target }); + return platform.devices[device.id].memory(kind) orelse stdx.debug.panic("Device {d} doesn't have {t} memory", .{ device.id, kind }); +} + fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { - var sh = shape; - if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; - return Sharding.Placement.init(&sh) catch |err| { + const placed = platform.placedShape(shape); + return Sharding.Placement.init(&placed) catch |err| { @branchHint(.cold); std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); }; diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig index b3e82a9c..20d90657 100644 --- a/zml/buffer/HostAccessible.zig +++ b/zml/buffer/HostAccessible.zig @@ -17,6 +17,7 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("../buffer.zig").Buffer; +const deviceMemory = @import("../buffer.zig").deviceMemory; const PjrtBufferMetadata = @import("../buffer.zig").PjrtBufferMetadata; const Platform = @import("../platform.zig").Platform; const Shape = @import("../shape.zig").Shape; @@ -249,7 +250,7 @@ pub fn init( ._shards = .empty, }; const metadata: PjrtBufferMetadata = .init(platform, shape_); - const devices = platform.physical_mesh.devices_in_canonical_order; + const devices = metadata.placement.mesh().?.devicesInCanonicalOrder(); const api = platform.pjrt_api; var allocations: HostPinnedAllocations = .empty; @@ -259,7 +260,7 @@ pub fn init( errdefer allocations.deinitHandles(api, buffer._shards.constSlice()); for (devices, 0..) |device, shard_index| { - const memory = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; + const memory = deviceMemory(platform, device, .host_pinned).pjrt_memory; const origin: Coordinates = .ofShard(&metadata.placement, shard_index, metadata.shape); const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.localShape(metadata.shape)) else null; diff --git a/zml/testing.zig b/zml/testing.zig index 3f115f51..023f937a 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -7,6 +7,8 @@ const Platform = @import("platform.zig").Platform; const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; const zml = @import("zml.zig"); +/// The new loader's delivery kinds; zml does not export its store. +const LoadMemory = @import("io/TensorStore.zig").Memory; const log = std.log.scoped(.@"zml/testing"); @@ -365,6 +367,51 @@ pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpec return error.TestExpectedEqual; } +/// `device_count` devices for a loader test: the accelerator the tests run +/// on (`env`, shared and never freed) when it has that many, otherwise a CPU +/// platform with exactly that many, freed by `deinit`. `physical` lays the +/// first `device_count` devices on one axis, so a mesh over it (`mesh`) +/// places on exactly those, whatever the platform has. Must not move after +/// `init`: meshes point at `physical`. +pub const Devices = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + owned: ?*Platform, + physical: Sharding.PhysicalMesh, + + pub fn init(self: *Devices, allocator: std.mem.Allocator, io: std.Io, device_count: usize) !void { + const shared = env(); + const owned: ?*Platform = if (shared.target != .cpu and shared.devices.len >= device_count) + null + else + try Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = @intCast(device_count) } }); + errdefer if (owned) |platform| platform.deinit(allocator, io); + const platform: *const Platform = if (owned) |p| p else shared; + const nodes = try allocator.alloc(Sharding.PhysicalNode, device_count); + defer allocator.free(nodes); + for (nodes, platform.devices[0..device_count]) |*node, device| node.* = .device(device); + self.* = .{ + .allocator = allocator, + .io = io, + .platform = platform, + .owned = owned, + .physical = try .fromTree(allocator, platform.target, .axis(.bus, .tree, nodes)), + }; + } + + pub fn deinit(self: *Devices) void { + self.physical.deinit(self.allocator); + if (self.owned) |platform| platform.deinit(self.allocator, self.io); + self.* = undefined; + } + + /// A mesh named `name` whose one axis, `model`, spans the devices. + pub fn mesh(self: *const Devices, name: []const u8) !Sharding.Mesh { + return .init(name, &self.physical, .mesh(.{ .model = .high_bandwidth }), .parseBindings(.{ .model = .bus })); + } +}; + pub const TensorBytes = struct { name: []const u8, /// The safetensors dtype name, such as "U16". @@ -430,6 +477,52 @@ pub fn pattern(bytes: []u8, seed: u8) void { for (bytes, 0..) |*byte, index| byte.* = (@as(u8, @truncate(index *% 29)) +% seed *% 53) | 1; } +/// Reads every shard of `buffer` back, in its mesh's device order, and +/// checks that it lives in `memory`'s space and holds that device's box of +/// `global`, the whole tensor's row-major bytes (sub-byte elements packed). +/// A buffer without a mesh is replicated on the platform's devices. On +/// CPU/neuron/metal every memory kind is the one memory space, so only +/// CUDA/ROCm/oneAPI tell them apart. +pub fn expectShards(allocator: std.mem.Allocator, io: std.Io, buffer: *const zml.Buffer, global: []const u8, memory: LoadMemory) !void { + const platform = buffer._platform; + const shape = platform.placedShape(buffer.shape()).packedShape(); + try std.testing.expectEqual(shape.byteSize(), global.len); + const placement: Sharding.Placement = try .init(&shape); + const devices = placement.mesh().?.devicesInCanonicalOrder(); + try std.testing.expectEqual(devices.len, buffer._shards.len); + const shard_bytes = placement.localShape(shape).byteSize(); + const actual = try allocator.alloc(u8, shard_bytes); + defer allocator.free(actual); + const expected = try allocator.alloc(u8, shard_bytes); + defer allocator.free(expected); + const rank = shape.rank(); + const strides = shape.computeByteStrides(); + var shards = buffer.shards(); + for (devices, 0..) |device, device_index| { + const shard = shards.next().?; + try std.testing.expect(shard._pjrt_buffer.memory(platform.pjrt_api) == platform.devices[device.id].memory(memory.space()).?.pjrt_memory); + try shard.toHost(io, actual); + // The device's box, one run of the last axis per index of the others. + const box = placement.slices(device_index, shape); + const run: usize = if (rank == 0) shape.dtype().sizeOf() else @intCast(box.get(rank - 1).size * strides.get(rank - 1)); + var index: [zml.Shape.MAX_RANK]i64 = @splat(0); + var cursor: usize = 0; + while (cursor < expected.len) : (cursor += run) { + var offset: i64 = 0; + for (0..rank) |axis| offset += (box.get(axis).start + index[axis]) * strides.get(axis); + @memcpy(expected[cursor..][0..run], global[@intCast(offset)..][0..run]); + var axis = rank -| 1; + while (axis > 0) { + axis -= 1; + index[axis] += 1; + if (index[axis] < box.get(axis).size) break; + index[axis] = 0; + } + } + try std.testing.expectEqualSlices(u8, expected, actual); + } +} + fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { var count: usize = 0; var it = store.store.registry.iterator(); From ca557bdaa68e69096bebf236450f5348e973914e Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:42:35 +0100 Subject: [PATCH 042/103] zml/io: add the loader's seam types zml/io/backend.zig holds the contract between the new loader's front end and its backends, which later changes build on from both sides. Nothing calls it yet; io.zig's test block reaches it. A LoadSpec is one destination: its shape, the memory it is delivered to, the caller's output buffer and the parts that fill it. A LoadPart is the bytes of one checkpoint tensor that a Lattice moves: a run of bytes, repeated over up to 2 * MAX_RANK dimensions with a source and a destination stride each, a zero source stride repeating the same bytes. Lattice.Cursor walks a range of runs without a division per run, and copyRuns copies them between two addresses. checkParts is the check the front end runs before a backend sees a spec: at least one part, each non-empty and inside its source and the destination, in ascending destination order, strided runs landing in index order, interleaving strided parts covering no more than the destination, and every file end far enough below maxInt(u64) that a read widened to the direct-I/O alignment does not overflow. Spans are checked for overflow before the order, so a part with huge strides is refused instead of overflowing. requestSize gives the source request for a DMA block: the read chunk rounded up to whole blocks, at least one, at most 32 MiB (max_read_request_size). A block that is not a nonzero multiple of the direct-I/O alignment, or a request too short to hold a read widened at both ends, has none. readAlignment is that alignment when the io is a VFS and direct I/O is not off, else 0. backend.Options carries the checked loader options; its dmaBlockSizes is the pinned block, else one fixed 16 MiB block. Co-Authored-By: Claude Opus 5.5 --- zml/BUILD.bazel | 1 + zml/io.zig | 1 + zml/io/backend.zig | 388 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 390 insertions(+) create mode 100644 zml/io/backend.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index cee65d79..5323b877 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -88,6 +88,7 @@ zig_library( "io.zig", "io/ShardGrid.zig", "io/TensorStore.zig", + "io/backend.zig", "io/log.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", diff --git a/zml/io.zig b/zml/io.zig index 8afb9636..2b0f5d20 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -29,6 +29,7 @@ const log = std.log.scoped(.@"zml/io"); test { _ = @import("io/ShardGrid.zig"); _ = @import("io/TensorStore.zig"); + _ = @import("io/backend.zig"); _ = @import("io/log.zig"); } diff --git a/zml/io/backend.zig b/zml/io/backend.zig new file mode 100644 index 00000000..b58e21f6 --- /dev/null +++ b/zml/io/backend.zig @@ -0,0 +1,388 @@ +//! The seam between the loader's front end and its backends: the +//! `LoadSpec` contract they trust, its parts' byte runs (`Lattice`), and the +//! source request size (`requestSize`). +const std = @import("std"); + +const VFS = @import("vfs"); + +const Buffer = @import("../buffer.zig").Buffer; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Memory = @import("TensorStore.zig").Memory; +const constants = @import("../constants.zig"); + +/// The largest source request, and so the largest DMA block: a request is +/// a whole number of blocks, at least one. +pub const max_read_request_size: usize = 32 * 1024 * 1024; + +/// The source request for DMA blocks of `block_size`: the read chunk +/// rounded up to whole blocks, one block at least. Null unless the block is +/// a nonzero multiple of `alignment` (`readAlignment`: reads land in +/// blocks), and the request is at most `max_read_request_size` and, with an +/// alignment, longer than two units, so a read widened by a unit at each +/// end fits. +pub fn requestSize(read_chunk_size: usize, block_size: usize, alignment: usize) ?usize { + if (block_size == 0 or block_size % @max(alignment, 1) != 0) return null; + // Bounded before the rounding, which then cannot overflow. + const least = @max(read_chunk_size, block_size); + if (least > max_read_request_size) return null; + const request = std.mem.alignForwardAnyAlign(usize, least, block_size); + if (request > max_read_request_size) return null; + if (alignment != 0 and request <= 2 * alignment) return null; + return request; +} + +/// What a backend widens the reads of a file read directly to: the VFS's +/// alignment when `io` is a VFS and `direct_io` is not `off`, else 0, every +/// read exact. The VFS decides per local file whether it is read directly +/// (`VFS.useDirectIo`). +pub fn readAlignment(io: std.Io, direct_io: VFS.DirectIo) usize { + if (direct_io == .off or VFS.fromIo(io) == null) return 0; + return VFS.direct_io_alignment; +} + +/// The loader's options as its front end checked them, for the backends. +pub const Options = struct { + /// Source reads in flight at once. + read_parallelism: usize, + /// The checkpoint's VFS backend's read hints. + read_hints: VFS.ReadHints = .local, + /// The DMA block, else `fixed_dma_block_size`. + dma_block_size: ?usize = null, + /// Whether local files are read directly (`VFS.useDirectIo`). + direct_io: VFS.DirectIo = .auto, + + /// The DMA block when the options pin none. + pub const fixed_dma_block_size: usize = 16 * 1024 * 1024; + + /// The blocks a backend may use: `dma_block_size`, else + /// `fixed_dma_block_size`. + pub fn dmaBlockSizes(self: *const Options) []const usize { + return if (self.dma_block_size) |*block_size| block_size[0..1] else &.{fixed_dma_block_size}; + } +}; + +/// One destination, its caller-owned output and the source ranges that fill +/// it, as the front end checked them (`checkParts`): backends trust them. +/// Everything a spec names (its parts, their dimensions, the sources, the +/// output) stays alive until its await; backends borrow it. +pub const LoadSpec = struct { + /// The destination's global shape, which the output buffer keeps. Its + /// sharding places the bytes on the devices; a shape without a mesh is + /// replicated over every device (`Platform.placedShape`). + shape: Shape, + /// Where the bytes go. A host kind comes only with contiguous parts + /// (`Lattice.dims` empty). + memory: Memory = .default, + /// Written whole once the destination's buffers exist, and not touched + /// before: a failure leaves it as it was or holding those buffers. + output: *Buffer, + /// What fills the destination's logical bytes, in ascending, disjoint + /// destination order; every byte no part covers is zero. + parts: []const LoadPart, +}; + +/// The bytes of `source` that `lattice` moves: its source offsets relative +/// to the tensor's data, its destination offsets into the destination's +/// logical bytes, the row-major bytes of its packed shape before the +/// sharding splits them. +pub const LoadPart = struct { + source: *const safetensors.Tensor, + lattice: Lattice, + + /// The whole of `source`, at the start of the destination. + pub fn whole(source: *const safetensors.Tensor) LoadPart { + return .{ .source = source, .lattice = .{ .width = source.byteSize() } }; + } +}; + +/// The most dimensions a part repeats over: `Source` gives each axis at +/// most two. +pub const max_part_dims = 2 * constants.MAX_RANK; + +/// A part's byte moves: the run of `width` bytes at `src_offset` and +/// `dst_offset`, and, for a strided part, one more for every index of +/// `dims` (outermost first), each offset moved by the index times its +/// strides. A zero source stride repeats the same bytes. Runs are numbered +/// row-major over `dims`. +pub const Lattice = struct { + src_offset: u64 = 0, + dst_offset: u64 = 0, + width: u64, + /// Empty for one contiguous run. + dims: []const Dim = &.{}, + + /// A step of the runs: `count` of them, each moving the source and + /// destination offsets by their strides. + pub const Dim = struct { + count: u64, + src_stride: u64, + dst_stride: u64, + }; + + pub const Run = struct { src: u64, dst: u64 }; + + pub fn runs(self: Lattice) u64 { + var n: u64 = 1; + for (self.dims) |d| n *= d.count; + return n; + } + + pub fn at(self: Lattice, index: u64) Run { + var rest = index; + var run: Run = .{ .src = self.src_offset, .dst = self.dst_offset }; + var d = self.dims.len; + while (d > 0) { + d -= 1; + const i = rest % self.dims[d].count; + rest /= self.dims[d].count; + run.src += i * self.dims[d].src_stride; + run.dst += i * self.dims[d].dst_stride; + } + return run; + } + + /// The source and destination bytes the runs span from their first + /// offsets, null when a count is zero or the bounds overflow. + pub fn spans(self: Lattice) ?struct { src: u64, dst: u64 } { + var src = self.width; + var dst = self.width; + for (self.dims) |d| { + if (d.count == 0) return null; + src = std.math.add(u64, std.math.mul(u64, d.count - 1, d.src_stride) catch return null, src) catch return null; + dst = std.math.add(u64, std.math.mul(u64, d.count - 1, d.dst_stride) catch return null, dst) catch return null; + } + return .{ .src = src, .dst = dst }; + } + + /// Whether each dimension steps past everything inside it on the + /// destination side: runs land in index order without overlapping. + /// Needs `spans` not null. + pub fn dstOrdered(self: Lattice) bool { + var extent = self.width; + var d = self.dims.len; + while (d > 0) { + d -= 1; + if (self.dims[d].count < 2) continue; + if (self.dims[d].dst_stride < extent) return false; + extent += (self.dims[d].count - 1) * self.dims[d].dst_stride; + } + return true; + } + + /// Runs `first` to `end` (exclusive), in index order, without a + /// division per run. + pub const Cursor = struct { + lattice: Lattice, + index: u64, + end: u64, + counters: [max_part_dims]u64, + run: Run, + + pub fn init(lattice: Lattice, first: u64, end: u64) Cursor { + std.debug.assert(lattice.dims.len <= max_part_dims); + var cursor: Cursor = .{ .lattice = lattice, .index = first, .end = end, .counters = undefined, .run = lattice.at(first) }; + var rest = first; + var d = lattice.dims.len; + while (d > 0) { + d -= 1; + cursor.counters[d] = rest % lattice.dims[d].count; + rest /= lattice.dims[d].count; + } + return cursor; + } + + pub fn next(self: *Cursor) ?Run { + if (self.index >= self.end) return null; + const current = self.run; + self.index += 1; + if (self.index < self.end) { + const dims = self.lattice.dims; + var d = dims.len; + while (d > 0) { + d -= 1; + self.counters[d] += 1; + self.run.src +%= dims[d].src_stride; + self.run.dst +%= dims[d].dst_stride; + if (self.counters[d] < dims[d].count) break; + self.counters[d] = 0; + self.run.src -%= dims[d].count * dims[d].src_stride; + self.run.dst -%= dims[d].count * dims[d].dst_stride; + } + } + return current; + } + }; + + /// Copies runs `first` to `end` (exclusive): the `width` bytes at + /// `src + run.src` to `dst + run.dst`, `src` and `dst` being the + /// addresses the offsets count from (wrapping). + pub fn copyRuns(self: Lattice, first: u64, end: u64, dst: usize, src: usize) void { + if (first >= end) return; + const width: usize = @intCast(self.width); + var cursor: Cursor = .init(self, first, end); + while (cursor.next()) |run| { + @memcpy(@as([*]u8, @ptrFromInt(dst +% run.dst))[0..width], @as([*]const u8, @ptrFromInt(src +% run.src))[0..width]); + } + } +}; + +/// Whether `parts` keep the `LoadSpec` contract for a destination of +/// `destination_bytes`: there is at least one, each is non-empty and inside +/// its source and the destination, and they come in ascending destination +/// order, each starting at or after the previous contiguous part's end or +/// strided part's start. Each part's bytes end in their file at least +/// `VFS.direct_io_alignment` below `maxInt(u64)`, so a read of them widened +/// to the alignment does not overflow. Strided parts must land in index +/// order (`Lattice.dstOrdered`); interleaving ones are only checked against +/// each other by the bytes they cover together, since only `Source` makes +/// them, disjoint by construction. +pub fn checkParts(parts: []const LoadPart, destination_bytes: u64) bool { + if (parts.len == 0) return false; + var floor: u64 = 0; + var covered: u64 = 0; + for (parts) |part| { + const lattice = part.lattice; + if (lattice.width == 0 or lattice.dims.len > max_part_dims) return false; + const spans = lattice.spans() orelse return false; + if (!lattice.dstOrdered()) return false; + const source_end = std.math.add(u64, lattice.src_offset, spans.src) catch return false; + const file_end = std.math.add(u64, part.source.offset, source_end) catch return false; + const end = std.math.add(u64, lattice.dst_offset, spans.dst) catch return false; + if (source_end > part.source.byteSize() or file_end > std.math.maxInt(u64) - VFS.direct_io_alignment) return false; + if (end > destination_bytes or lattice.dst_offset < floor) return false; + floor = if (lattice.dims.len == 0) end else lattice.dst_offset; + covered +|= lattice.runs() * lattice.width; + } + return covered <= destination_bytes; +} + +/// A random lattice whose runs land in order (`Lattice.dstOrdered`), its +/// source strides anything, zero (a broadcast) included. +fn randomLattice(random: std.Random, dims: []Lattice.Dim) Lattice { + const width = random.intRangeAtMost(u64, 1, 17); + var extent = width; + var d = dims.len; + while (d > 0) { + d -= 1; + dims[d] = .{ + .count = random.intRangeAtMost(u64, 1, 5), + .src_stride = random.intRangeAtMost(u64, 0, 40), + .dst_stride = extent + random.intRangeAtMost(u64, 0, 3) * random.intRangeAtMost(u64, 0, 4), + }; + extent = dims[d].dst_stride * (dims[d].count - 1) + extent; + } + return .{ .src_offset = random.intRangeAtMost(u64, 0, 9), .dst_offset = random.intRangeAtMost(u64, 0, 9), .width = width, .dims = dims }; +} + +test "the cursor walks a range's runs where at places them, and copyRuns copies them" { + var prng: std.Random.DefaultPrng = .init(0x5eed); + const random = prng.random(); + var source: [4096]u8 = undefined; + for (&source, 0..) |*byte, i| byte.* = @truncate(i * 7 + 1); + var expected: [4096]u8 = undefined; + var actual: [4096]u8 = undefined; + var dims_buffer: [5]Lattice.Dim = undefined; + for (0..2000) |_| { + const lattice = randomLattice(random, dims_buffer[0..random.intRangeAtMost(usize, 0, dims_buffer.len)]); + const spans = lattice.spans().?; + if (lattice.src_offset + spans.src > source.len or lattice.dst_offset + spans.dst > expected.len) continue; + try std.testing.expect(lattice.dstOrdered()); + const runs = lattice.runs(); + const first = random.uintAtMost(u64, runs); + const end = random.intRangeAtMost(u64, first, runs); + @memset(&expected, 0); + @memset(&actual, 0); + var cursor: Lattice.Cursor = .init(lattice, first, end); + for (first..end) |index| { + const run = lattice.at(index); + try std.testing.expectEqual(run, cursor.next().?); + const width: usize = @intCast(lattice.width); + @memcpy(expected[@intCast(run.dst)..][0..width], source[@intCast(run.src)..][0..width]); + } + try std.testing.expectEqual(null, cursor.next()); + lattice.copyRuns(first, end, @intFromPtr(&actual), @intFromPtr(&source)); + try std.testing.expectEqualSlices(u8, &expected, &actual); + } +} + +test "checkParts refuses parts outside their source or destination, out of order or overlapping" { + // A source of 16 bytes into a destination of 32. + const source: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = 0 }; + // Sixteen bytes ending at the largest file offset a read widened to the + // alignment can reach, and one byte later. + const last: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = std.math.maxInt(u64) - VFS.direct_io_alignment - 16 }; + const past_last: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = last.offset + 1 }; + // Runs of 4 bytes, every other 4 bytes of the destination. + const every_other: []const Lattice.Dim = &.{.{ .count = 4, .src_stride = 4, .dst_stride = 8 }}; + const refused = [_][]const LoadPart{ + &.{}, + &.{.{ .source = &source, .lattice = .{ .width = 0 } }}, + // Past the source, past the destination, an overflow, and past the + // largest file offset. + &.{.{ .source = &source, .lattice = .{ .src_offset = 8, .width = 9 } }}, + &.{.{ .source = &source, .lattice = .{ .dst_offset = 24, .width = 9 } }}, + &.{.{ .source = &source, .lattice = .{ .src_offset = std.math.maxInt(u64), .width = 1 } }}, + &.{.whole(&past_last)}, + // Overlapping, and descending. + &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 8 } } }, + &.{ .{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4 } }, .{ .source = &source, .lattice = .{ .width = 4 } } }, + // Strided: runs that overlap on the destination, a dimension of no + // step, strides whose span overflows, past the source, past the + // destination, and descending. + &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 2 }} } }}, + &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 0, .src_stride = 4, .dst_stride = 8 }} } }}, + &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 1 << 40, .src_stride = 0, .dst_stride = 1 << 40 }} } }}, + &.{.{ .source = &source, .lattice = .{ .src_offset = 4, .width = 4, .dims = every_other } }}, + &.{.{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4, .dims = every_other } }}, + &.{ .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } } }, + // Interleaving strided parts that cover more than the destination, + // and contiguous parts overlapping beside a strided one. + &.{ .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } } }, + &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 16, .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 8 }} } } }, + }; + for (refused) |parts| try std.testing.expect(!checkParts(parts, 32)); + const accepted = [_][]const LoadPart{ + // Back to back. + &.{ .{ .source = &source, .lattice = .{ .width = 16 } }, .{ .source = &source, .lattice = .{ .dst_offset = 16, .width = 16 } } }, + // Two strided parts interleaving their runs. + &.{ .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } } }, + // A contiguous part, then a strided one. + &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 8 }} } } }, + // A broadcast: one source run repeated over the destination. + &.{.{ .source = &source, .lattice = .{ .width = 8, .dims = &.{.{ .count = 4, .src_stride = 0, .dst_stride = 8 }} } }}, + // Ending at the largest file offset. + &.{.whole(&last)}, + }; + for (accepted) |parts| try std.testing.expect(checkParts(parts, 32)); +} + +test "requestSize rounds the read chunk up to whole blocks of whole alignment units" { + const mib: usize = 1 << 20; + // One block at least, the chunk rounded up to whole blocks, and a + // rounded request past the largest refused. + try std.testing.expectEqual(16 * mib, requestSize(8 * mib, 16 * mib, 4096)); + try std.testing.expectEqual(9 * mib, requestSize(8 * mib, 3 * mib, 4096)); + try std.testing.expectEqual(null, requestSize(32 * mib, 24 * mib, 0)); + // Without an alignment any nonzero block; with one, only its multiples. + try std.testing.expectEqual(8 * mib + 2048, requestSize(8 * mib, 8 * mib + 2048, 0)); + try std.testing.expectEqual(null, requestSize(8 * mib, 8 * mib + 2048, 4096)); + try std.testing.expectEqual(null, requestSize(8 * mib, 0, 0)); + // A request of two alignment units cannot hold a read widened at both + // ends. + try std.testing.expectEqual(null, requestSize(4096, 8192, 4096)); + try std.testing.expectEqual(3 * 4096, requestSize(4096, 3 * 4096, 4096)); +} + +test "the options pin a DMA block or take the fixed one, and only a VFS widens reads" { + const pinned: Options = .{ .read_parallelism = 1, .dma_block_size = 4 << 20 }; + try std.testing.expectEqualSlices(usize, &.{4 << 20}, pinned.dmaBlockSizes()); + const fixed: Options = .{ .read_parallelism = 1 }; + try std.testing.expectEqualSlices(usize, &.{Options.fixed_dma_block_size}, fixed.dmaBlockSizes()); + + var filesystem = try VFS.init(std.testing.allocator, std.testing.io); + defer filesystem.deinit(); + try std.testing.expectEqual(VFS.direct_io_alignment, readAlignment(filesystem.io(), .auto)); + try std.testing.expectEqual(0, readAlignment(filesystem.io(), .off)); + try std.testing.expectEqual(0, readAlignment(std.testing.io, .on)); +} From eaa5ce324f6c8f591d9a3ae67ffb8a3e124c1608 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:48:36 +0100 Subject: [PATCH 043/103] zml/io: add Source: checkpoint bits as lattices, reshape-only ops, placement last, one lowering rule zml/io/Source.zig describes a tensor the new loader assembles by moving checkpoint bytes, never by computing them. A source is a shape and a list of pieces: each piece reads the bits of one checkpoint tensor through an affine lattice, mapping the source's row-major elements to bit offsets in the tensor, and elements no piece covers are zero. Pieces live in the store's arena. This change adds the source of one checkpoint tensor (init through a View), its accessors, and the ops that only relabel or reshape it (withTags, rename, renameTag, splitAxis, merge, reshape, flatten, insertAxes, appendAxes, squeeze), which keep the row-major order and so never touch a piece. Placement comes last: withPartitioning and as return a Placed, the only thing that binds. Placed.bind records the model tensor through a private bindNew, the one writer of bindings, so pieces stay disjoint by construction. Placed.lower turns each piece into one backend.LoadPart by one rule: dimensions outermost first, the innermost run contiguous on both sides as the width, and SubBytePosition for a run that starts or ends inside a byte. withPartitioning(.replicated, ...) takes only .{}, checked at compile time. The tests replay a binding's lowered parts on the host and fail on any byte written twice. Nothing imports the file yet: the store wiring that binds through it follows. Co-Authored-By: Claude Opus 5.5 --- zml/BUILD.bazel | 1 + zml/io/Source.zig | 420 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+) create mode 100644 zml/io/Source.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 5323b877..309c0213 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -87,6 +87,7 @@ zig_library( "floats.zig", "io.zig", "io/ShardGrid.zig", + "io/Source.zig", "io/TensorStore.zig", "io/backend.zig", "io/log.zig", diff --git a/zml/io/Source.zig b/zml/io/Source.zig new file mode 100644 index 00000000..1fc93c47 --- /dev/null +++ b/zml/io/Source.zig @@ -0,0 +1,420 @@ +//! A tensor the loader assembles from checkpoint bytes by moving them, never +//! by computing: a checkpoint tensor, its axes relabeled or reshaped. The +//! ops are `zml.Tensor`'s shape ops, with the same names, tags and argument +//! conventions. +//! +//! A source comes from its store (`View.source`) and its pieces live in +//! the store's arena. Nothing is freed before `TensorStore.deinit`, and the +//! store must not move while sources point to it. The ops touch neither the +//! registry nor the bindings. Placement comes last: `withPartitioning` and +//! `as` return a `Placed`, which has no ops and drops its placement with +//! `unplaced`. A `Placed` is what binds: `Placed.bind` records the model +//! tensor the loader fills from it. Sources are built during model init. +const std = @import("std"); + +const stdx = @import("stdx"); + +const constants = @import("../constants.zig"); +const DataType = @import("../dtype.zig").DataType; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; +const backend = @import("backend.zig"); +const TensorStore = @import("TensorStore.zig"); + +const Source = @This(); + +store: *TensorStore, +/// Always replicated: placement comes last (`Placed`). +_shape: Shape, +/// Disjoint lattices of elements read from checkpoint tensors; elements no +/// piece covers are zero. zml/io's own: a binding keeps them (`Placed`) +/// and the loader lowers them to bytes (`Placed.lower`). +_pieces: []const Piece, + +/// Checkpoint bits read through an affine lattice: for every `j` with +/// `j[k] < dims[k].count`, the element at row-major index +/// `origin + sum(j[k] * dims[k].step)` of the source's shape is the element +/// at bit `offset + sum(j[k] * dims[k].stride)` of `tensor`. Reshapes keep +/// the row-major order, so they never touch a piece. +const Piece = struct { + tensor: *const safetensors.Tensor, + offset: u64, + origin: u64, + /// Outermost first, none of count 1. + dims: []const Dim, +}; + +const Dim = struct { + count: u64, + /// Row-major elements of the source's shape between two steps. + step: u64, + /// Bits of the checkpoint tensor between two steps; zero repeats them. + stride: u64, +}; + +/// The checkpoint tensor `subkey` of `view`, with `tags` (null keeps the +/// checkpoint's); null when the checkpoint has no such tensor. +pub fn init(view: TensorStore.View, subkey: []const u8, tags: ?[]const Shape.Tag) ?Source { + const tensor = view.get(subkey) orelse return null; + const store = view.store; + var shape_ = tensor.shape; + if (tags) |user_tags| { + stdx.debug.assert(user_tags.len == shape_.rank(), "checkpoint tensor {s} has shape {f}, but got {d} tags: {f}", .{ tensor.name, shape_, user_tags.len, stdx.fmt.stringsZ(user_tags) }); + @memcpy(shape_._tags.slice(), user_tags); + } + shape_._sharding = .replicated; + const arena = store.arena.allocator(); + const elements: u64 = shape_.count(); + const pieces: []const Piece = if (elements == 0) &.{} else blk: { + const dims_: []const Dim = if (elements == 1) + &.{} + else + arena.dupe(Dim, &.{.{ .count = elements, .step = 1, .stride = shape_.dtype().bitSizeOf() }}) catch oom(); + break :blk arena.dupe(Piece, &.{.{ .tensor = tensor, .offset = 0, .origin = 0, .dims = dims_ }}) catch oom(); + }; + return .{ .store = store, ._shape = shape_, ._pieces = pieces }; +} + +pub fn shape(self: Source) Shape { + return self._shape; +} + +pub fn dim(self: Source, axis_: anytype) i64 { + return self._shape.dim(axis_); +} + +pub fn dims(self: *const Source) []const i64 { + return self._shape.dims(); +} + +pub fn dtype(self: Source) DataType { + return self._shape.dtype(); +} + +pub fn rank(self: Source) u4 { + return self._shape.rank(); +} + +pub fn axis(self: Source, axis_: anytype) u3 { + return self._shape.axis(axis_); +} + +pub fn count(self: Source) usize { + return self._shape.count(); +} + +pub fn format(self: Source, writer: *std.Io.Writer) !void { + try writer.print("Source({f}, {d} pieces)", .{ self._shape, self._pieces.len }); +} + +// Reshapes: the row-major order is unchanged, so are the pieces. + +pub fn withTags(self: Source, tags: anytype) Source { + return self.reshaped(self._shape.withTags(tags)); +} + +pub fn rename(self: Source, renames: anytype) Source { + return self.reshaped(self._shape.rename(renames)); +} + +pub fn renameTag(self: Source, old: anytype, new: anytype) Source { + return self.reshaped(self._shape.setTag(old, new)); +} + +pub fn splitAxis(self: Source, ax: anytype, split_shape: anytype) Source { + return self.reshaped(self._shape.splitAxis(ax, split_shape)); +} + +pub fn merge(self: Source, merges: anytype) Source { + return self.reshaped(self._shape.mergeAxes(merges)); +} + +pub fn reshape(self: Source, output_shape: anytype) Source { + return self.reshaped(self._shape.reshape(output_shape)); +} + +pub fn flatten(self: Source) Source { + return self.reshape(.{self.count()}); +} + +pub fn insertAxes(self: Source, axis_: anytype, tags: anytype) Source { + const tags_ = Shape.parseTags(tags); + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); + var res = self._shape; + const ones: [constants.MAX_RANK]i64 = @splat(1); + res._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; + res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + return self.reshaped(res); +} + +pub fn appendAxes(self: Source, tags: anytype) Source { + return self.insertAxes(.last, tags); +} + +pub fn squeeze(self: Source, axis_: anytype) Source { + const a = self.axis(axis_); + stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); + return self.reshaped(self._shape.remove(a)); +} + +fn reshaped(self: Source, new_shape: Shape) Source { + stdx.debug.assert(new_shape.count() == self.count(), "Source reshape from {f} to {f} changes the element count", .{ self._shape, new_shape }); + var res = self; + res._shape = new_shape.withDtype(self.dtype()); + res._shape._sharding = .replicated; + return res; +} + +/// This source placed on `mesh_` by `partitioning`: a struct keyed by the +/// source's tags (axes it does not name are replicated), or a +/// `Sharding.Partitioning`. `.replicated` places it on every device without +/// a mesh, and takes only `.{}`. Placement comes last: a `Placed` only +/// binds (`Placed.bind`) or drops its placement (`Placed.unplaced`). +pub fn withPartitioning(self: Source, comptime mesh_: @EnumLiteral(), partitioning: anytype) Placed { + var placed = self._shape; + if (mesh_ == .replicated) { + const T = @TypeOf(partitioning); + if (comptime !(stdx.meta.isStruct(T) and std.meta.fieldNames(T).len == 0)) + @compileError("Source.withPartitioning(.replicated, ...) takes .{}: a replicated source has no partitioning, got " ++ @typeName(T)); + } else { + placed = self._shape.withPartitioning(self.store.mesh(mesh_), partitioning); + } + return .{ .store = self.store, .shape = placed, ._pieces = self._pieces }; +} + +/// This source in `placed`, a shape built elsewhere: its dimensions, tags +/// and sharding. Like `reshape`, it moves no element: `placed` has the +/// source's element count and dtype. Placement comes last, as for +/// `withPartitioning`. +pub fn as(self: Source, placed: Shape) Placed { + stdx.debug.assert(placed.dtype() == self.dtype() and placed.count() == self.count(), "Source {f} cannot be placed as {f}", .{ self._shape, placed }); + return .{ .store = self.store, .shape = placed, ._pieces = self._pieces }; +} + +/// A source with its placement (`withPartitioning`, `as`): the only thing +/// that binds, and what a binding keeps (`TensorStore.Binding`). +pub const Placed = struct { + store: *TensorStore, + /// The placed shape, whose row-major elements the pieces index. + shape: Shape, + /// See `Source._pieces`. + _pieces: []const Piece, + + /// Makes the model tensor the loader fills from this source, in default + /// memory. The binding keeps the source as it is; the loader lowers it + /// to bytes (`lower`). + pub fn bind(self: Placed) Tensor { + return bindNew(self.store, self.shape, .{ .load = .{ .source = self } }); + } + + /// This source without its placement, for code that takes a placed + /// source and reworks it by its placement (a layout per device), then + /// places the result again. + pub fn unplaced(self: Placed) Source { + var shape_ = self.shape; + shape_._sharding = .replicated; + return .{ .store = self.store, ._shape = shape_, ._pieces = self._pieces }; + } + + /// The checkpoint tensor this source is, whole and alone, if it is: one + /// piece that reads all of its bits in order. + pub fn whole(self: Placed) ?*const safetensors.Tensor { + if (self._pieces.len != 1) return null; + const piece = self._pieces[0]; + if (piece.offset != 0 or piece.origin != 0 or piece.tensor.byteSize() != self.shape.byteSize()) return null; + const contiguous = switch (piece.dims.len) { + 0 => self.shape.count() == 1, + 1 => piece.dims[0].count == self.shape.count() and piece.dims[0].step == 1 and piece.dims[0].stride == self.shape.dtype().bitSizeOf(), + else => false, + }; + return if (contiguous) piece.tensor else null; + } + + /// The bytes the loader moves, one part per piece in piece order, in + /// `arena`; none for a source that reads no bytes. The one rule from a + /// source's bits to bytes: each part's dimensions outermost first, the + /// innermost run that is contiguous on both sides as its width. + /// `error.SubBytePosition` when a run starts or ends inside a byte. + pub fn lower(self: Placed, arena: std.mem.Allocator) error{ OutOfMemory, SubBytePosition }![]backend.LoadPart { + const parts = try arena.alloc(backend.LoadPart, self._pieces.len); + for (parts, self._pieces) |*part, piece| part.* = try lowerPiece(arena, self.shape, piece); + return parts; + } +}; + +/// A new tensor of `shape_`, bound to `binding`: the one maker of +/// `store`'s bindings (`Placed.bind`), whose pieces are disjoint by +/// construction: the loader relies on that and does not check. +fn bindNew(store: *TensorStore, shape_: Shape, binding: TensorStore.Binding) Tensor { + const tensor: Tensor = .fromShape(shape_); + store.id_to_sources.putNoClobber(store.allocator, tensor.id, binding) catch oom(); + return tensor; +} + +/// `piece` of a source placed in `shape_` as one part (`Placed.lower`), its +/// dimensions allocated with `allocator`, none for one run. +fn lowerPiece(allocator: std.mem.Allocator, shape_: Shape, piece: Piece) error{ OutOfMemory, SubBytePosition }!backend.LoadPart { + const ebits: u64 = shape_.dtype().bitSizeOf(); + var lowered: stdx.BoundedArray(backend.Lattice.Dim, backend.max_part_dims) = .empty; + for (piece.dims) |d| lowered.appendAssumeCapacity(.{ .count = d.count, .src_stride = d.stride, .dst_stride = d.step * ebits }); + var width: u64 = ebits; + if (lowered.len != 0) { + const inner = lowered.get(lowered.len - 1); + if (inner.dst_stride == ebits and inner.src_stride == ebits) { + width = inner.count * ebits; + _ = lowered.pop(); + } + } + const dst_bits = piece.origin * ebits; + var aligned = piece.offset % 8 == 0 and dst_bits % 8 == 0 and width % 8 == 0; + for (lowered.slice()) |*d| { + aligned = aligned and d.src_stride % 8 == 0 and d.dst_stride % 8 == 0; + d.src_stride /= 8; + d.dst_stride /= 8; + } + if (!aligned) return error.SubBytePosition; + return .{ .source = piece.tensor, .lattice = .{ + .src_offset = piece.offset / 8, + .dst_offset = dst_bits / 8, + .width = width / 8, + .dims = if (lowered.len == 0) &.{} else try allocator.dupe(backend.Lattice.Dim, lowered.constSlice()), + } }; +} + +fn oom() noreturn { + @panic("OOM"); +} + +// Tests: the loader's byte placement of a binding, replayed on the host. + +/// A checkpoint of in-memory tensors (the registry points nowhere) and the +/// bytes each holds. +const TestCheckpoint = struct { + store: TensorStore, + bytes: std.StringHashMapUnmanaged([]u8) = .empty, + allocator: std.mem.Allocator, + + const Entry = struct { name: []const u8, shape: Shape }; + + /// Registers `entries` end to end, each holding distinct bytes. + fn init(self: *TestCheckpoint, allocator: std.mem.Allocator, entries: []const Entry) !void { + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var offset: u64 = 0; + for (entries) |entry| { + try registry.registerTensor(.{ .file_uri = "memory", .name = entry.name, .shape = entry.shape, .offset = offset }); + offset += entry.shape.byteSize(); + } + self.* = .{ .store = .init(allocator, ®istry, .{}), .allocator = allocator }; + errdefer self.deinit(); + for (entries, 0..) |entry, index| { + const bytes = try allocator.alloc(u8, entry.shape.byteSize()); + errdefer allocator.free(bytes); + for (bytes, 0..) |*byte, i| byte.* = @truncate(i *% 7 +% index *% 61 +% 1); + try self.bytes.put(allocator, self.store.registry.tensors.getKey(entry.name).?, bytes); + } + } + + fn deinit(self: *TestCheckpoint) void { + var it = self.bytes.valueIterator(); + while (it.next()) |bytes| self.allocator.free(bytes.*); + self.bytes.deinit(self.allocator); + self.store.deinit(); + } + + /// The bytes the loader would assemble for `tensor`, placed on the host: + /// its source lowered as the loader lowers it (`Placed.lower`), the + /// bytes no part covers zero. Fails when a part writes a byte another + /// wrote, so that every byte is written exactly once, by a part or the + /// zero fill: the loader relies on disjoint pieces and does not check + /// them. + fn assemble(self: *TestCheckpoint, tensor: Tensor) ![]u8 { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + const parts = try self.store.bindingOf(tensor.id).?.load.source.lower(arena.allocator()); + const out = try self.allocator.alloc(u8, tensor.shape().byteSize()); + errdefer self.allocator.free(out); + @memset(out, 0); + const written = try arena.allocator().alloc(bool, out.len); + @memset(written, false); + for (parts) |part| { + const src = self.bytes.get(part.source.name).?; + const width: usize = @intCast(part.lattice.width); + var runs: backend.Lattice.Cursor = .init(part.lattice, 0, part.lattice.runs()); + while (runs.next()) |run| { + const at: usize = @intCast(run.dst); + for (written[at..][0..width]) |*byte| { + if (byte.*) return error.DoubleWrite; + byte.* = true; + } + @memcpy(out[at..][0..width], src[@intCast(run.src)..][0..width]); + } + } + return out; + } +}; + +test "a source binds as its pieces, which the loader lowers to byte parts" { + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(std.testing.allocator, &.{ + .{ .name = "q", .shape = .init(.{ 4, 3 }, .u16) }, + }); + defer checkpoint.deinit(); + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const view = checkpoint.store.view(); + const q = view.source("q", .{ .row, .col }); + const entries = [_]*const safetensors.Tensor{view.get("q").?}; + + // Relabels keep the whole tensor, one run of it; only `bind` records + // anything, and the binding keeps the placed source as it is. + const relabeled = q.splitAxis(.row, .{ .h = 2, .hd = 2 }).merge(.{ .row = .{ .h, .hd } }).withPartitioning(.replicated, .{}); + try std.testing.expectEqual(@as(usize, 0), checkpoint.store.id_to_sources.count()); + const bound = checkpoint.store.bindingOf(relabeled.bind().id).?.load.source; + try std.testing.expectEqual(relabeled._pieces, bound._pieces); + try std.testing.expectEqual(entries[0], bound.whole().?); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{.whole(entries[0])}, try bound.lower(arena.allocator())); +} + +test "assembling fails on a byte two pieces write" { + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(std.testing.allocator, &.{.{ .name = "q", .shape = .init(.{ 4, 3 }, .u16) }}); + defer checkpoint.deinit(); + // Only a hand-built binding can repeat a piece. + const q = checkpoint.store.view().source("q", .{ .row, .col }); + const pieces = try checkpoint.store.arena.allocator().dupe(Piece, &.{ q._pieces[0], q._pieces[0] }); + const tensor = bindNew(&checkpoint.store, q.shape(), .{ .load = .{ .source = .{ .store = &checkpoint.store, .shape = q.shape(), ._pieces = pieces } } }); + try std.testing.expectError(error.DoubleWrite, checkpoint.assemble(tensor)); +} + +test "a source is placed last, by as or withPartitioning, and binds in its placed shape" { + const allocator = std.testing.allocator; + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(allocator, &.{.{ .name = "w", .shape = .init(.{ 4, 6 }, .u16) }}); + defer checkpoint.deinit(); + const mesh: Sharding.Mesh = .initTest(.tp, .{.model}); + const w = checkpoint.store.view().source("w", .{ .row, .col }); + + // `as` takes a shape built elsewhere, here of another rank: its + // dimensions, tags and sharding, for the same elements. + const placed = Shape.init(.{ .flat = 24 }, .u16).withPartitioning(&mesh, .{ .flat = .model }); + const placed_source = w.as(placed); + const flat = placed_source.bind(); + try std.testing.expect(flat.shape().eqlWithTagsAndPartitioning(placed)); + try std.testing.expect(checkpoint.store.bindingOf(flat.id).?.load.source.shape.eqlWithTagsAndPartitioning(placed)); + const replicated = w.withPartitioning(.replicated, .{}).bind(); + try std.testing.expect(replicated.shape().eqlWithTags(w.shape())); + try std.testing.expectEqual(null, replicated.shape()._sharding.mesh); + // `unplaced` keeps the placed dimensions and tags, without the sharding. + const dropped = placed_source.unplaced(); + try std.testing.expect(dropped.shape().eqlWithTags(placed)); + try std.testing.expectEqual(null, dropped.shape()._sharding.mesh); + + const flat_bytes = try checkpoint.assemble(flat); + defer allocator.free(flat_bytes); + const replicated_bytes = try checkpoint.assemble(replicated); + defer allocator.free(replicated_bytes); + try std.testing.expectEqualSlices(u8, replicated_bytes, flat_bytes); +} From 5f46a882a615bdf811f45c9ecd3f8a90b3dbb95e Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:49:11 +0100 Subject: [PATCH 044/103] zml/io: bind the new store's tensors through Source The new TensorStore now binds model tensors. A Binding is what the loader will read of a tensor: its load arm holds the placed source the tensor is filled from and the memory it is delivered to, default device memory. bindingOf looks a tensor's binding up by id; the store owns its bindings and every source in its arena, and frees them in deinit. View gains source and maybeSource, which give a checkpoint tensor as a Source with optional tags, and createTensor, maybeCreateTensor and createReplicatedTensor, which are source(...).withPartitioning(...).bind(). A replicated tensor has no mesh and takes only .{} as its partitioning. A missing key panics with the key, so binding cannot fail. io.zig's test block reaches Source.zig, whose tests now run. Co-Authored-By: Claude Opus 5.5 --- zml/io.zig | 1 + zml/io/TensorStore.zig | 102 +++++++++++++++++++++++++++++++++++------ 2 files changed, 88 insertions(+), 15 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index 2b0f5d20..20987cf1 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -28,6 +28,7 @@ const log = std.log.scoped(.@"zml/io"); // tests reach them, from here. test { _ = @import("io/ShardGrid.zig"); + _ = @import("io/Source.zig"); _ = @import("io/TensorStore.zig"); _ = @import("io/backend.zig"); _ = @import("io/log.zig"); diff --git a/zml/io/TensorStore.zig b/zml/io/TensorStore.zig index 969ed704..849729b3 100644 --- a/zml/io/TensorStore.zig +++ b/zml/io/TensorStore.zig @@ -1,11 +1,14 @@ -//! The new loader's store: prefixed views over the entries it takes from a -//! checkpoint registry. zml does not export it: models and the loader they -//! use build `zml.io.TensorStore` (io.zig), which borrows its registry, so -//! production changes go there and the new loader's here. +//! The new loader's store: it associates model tensors with checkpoint +//! sources and prefixed views, over the entries it takes from a checkpoint +//! registry. zml does not export it: models and the loader they use build +//! `zml.io.TensorStore` (io.zig), which borrows its registry, so production +//! changes go there and the new loader's here. //! -//! The store owns those entries. Views point to it: once `view` was called -//! it must not move, and it must outlive them. It borrows the meshes of -//! `Options.meshes`, which belong to the platform. +//! The store owns those entries, the bindings, and every source (its +//! arena). Views and sources point to it: once `view` was called it must +//! not move, and it must outlive them. It borrows the meshes of +//! `Options.meshes`, which belong to the platform. Binding cannot fail: a +//! missing key panics with the key. const std = @import("std"); const stdx = @import("stdx"); @@ -13,13 +16,15 @@ const stdx = @import("stdx"); const MemorySpace = @import("../platform.zig").Memory.Kind; const Quantization = @import("../quantization.zig").Quantization; const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; const TensorStore = @This(); /// Where the loader delivers a loaded tensor, on every device of its mesh. -/// The destination is allocated once per device; the host kinds take no -/// device memory. +/// The destination is allocated once per device whatever its pieces; the +/// host kinds take no device memory. pub const Memory = enum { /// Device memory. default, @@ -50,14 +55,32 @@ pub const Memory = enum { } }; +/// What the loader reads of a bound tensor (`bindingOf`): the source it +/// fills the tensor from. +pub const Binding = union(enum) { + /// The loader fills the tensor from `source` (`Source.Placed.bind`), + /// every byte no piece covers zero. + load: Load, + + pub const Load = struct { + source: Source.Placed, + /// Where the loader delivers this tensor. + memory: Memory = .default, + }; +}; + +pub const Source = @import("Source.zig"); + registry: safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh, quantization: ?Quantization.Target, +id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), +allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, pub const Options = struct { - /// The meshes `View.mesh` names. The slice is copied; the meshes must - /// outlive the store. + /// The meshes `Source.withPartitioning` and `View.mesh` name. The slice + /// is copied; the meshes must outlive the store. meshes: []const *const Sharding.Mesh = &.{}, /// How the platform's FP8 GEMMs want block-FP8 weights, for the code /// that binds them (`View.quantization`): `.of(platform)`. Required to @@ -77,6 +100,8 @@ pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, .registry = registry.*, .meshes = meshes, .quantization = opts.quantization, + .id_to_sources = .empty, + .allocator = allocator, .arena = arena, }; registry.* = .init(registry.arena.child_allocator); @@ -84,13 +109,21 @@ pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, return self; } -/// Frees the registry entries `init` took. +/// Frees the bindings and the sources, then the registry entries `init` +/// took. pub fn deinit(self: *TensorStore) void { + self.id_to_sources.deinit(self.allocator); self.arena.deinit(); self.registry.tensors.unlockPointers(); self.registry.deinit(); } +/// The binding of tensor `id`, as the loader reads it; null when this store +/// did not bind it. +pub fn bindingOf(self: *const TensorStore, id: Tensor.Id) ?Binding { + return self.id_to_sources.get(id); +} + /// The root view, with no prefix. It points to this store, which must not /// move afterwards. pub fn view(self: *TensorStore) View { @@ -113,8 +146,8 @@ pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mes } /// What model code holds: a key prefix over the store's checkpoint, to look -/// tensors up. A value that keeps its own copy of the prefix (256 bytes at -/// most, longer panics) and points to the store. +/// tensors up and bind them. A value that keeps its own copy of the prefix +/// (256 bytes at most, longer panics) and points to the store. pub const View = struct { store: *TensorStore, @@ -129,7 +162,8 @@ pub const View = struct { } /// This view under `prefix_` and a dot: `withPrefix("model")` then - /// `get("embed")` looks up `model.embed`. + /// `get("embed")` looks up `model.embed`. It only scopes lookups; + /// nothing is bound. pub fn withPrefix(self: *const View, prefix_: []const u8) View { var buffer: [256]u8 = undefined; const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix(), prefix_ }); @@ -167,6 +201,26 @@ pub const View = struct { return self.store.registry.tensors.getPtr(key); } + /// The checkpoint tensor `subkey`, with `tags` (null keeps the + /// checkpoint's), placed on `mesh_` by `partitioning` and bound: + /// `source(subkey, tags).withPartitioning(mesh_, partitioning).bind()`. + /// Panics when the checkpoint has no such tensor. + pub fn createTensor(self: View, subkey: []const u8, comptime tags: anytype, comptime mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); + } + + /// Like `createTensor`, but null when the checkpoint has no such tensor. + pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, comptime mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const source_ = self.maybeSource(subkey, tags) orelse return null; + return source_.withPartitioning(mesh_, partitioning).bind(); + } + + /// `createTensor` replicated on every device, without a mesh. + pub fn createReplicatedTensor(self: View, subkey: []const u8, comptime tags: anytype) Tensor { + return self.createTensor(subkey, tags, .replicated, .{}); + } + /// See `TensorStore.mesh`. pub fn mesh(self: View, name: @EnumLiteral()) *const Sharding.Mesh { return self.store.mesh(name); @@ -179,6 +233,20 @@ pub const View = struct { stdx.debug.panic("TensorStore.Options.quantization is unset: block-FP8 weights bind for a platform; pass .of(platform)", .{}); } + /// The checkpoint tensor `subkey` as a `Source`, with `tags` (null keeps + /// the checkpoint's). Panics when the checkpoint has no such tensor. + pub fn source(self: View, subkey: []const u8, comptime tags: anytype) Source { + return self.maybeSource(subkey, tags) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); + } + + /// Like `source`, but null when the checkpoint has no such tensor. + pub fn maybeSource(self: View, subkey: []const u8, comptime tags: anytype) ?Source { + const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); + const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + return .init(self, subkey, if (has_tags) parsed_tags.constSlice() else null); + } + /// The checkpoint tensors under this view's prefix: a scan of every key. pub fn count(self: View) usize { var count_: usize = 0; @@ -219,4 +287,8 @@ test "get finds exact keys, the registry's own entries, and count the keys under try std.testing.expectEqualStrings("layer.experts.0.", layer.withPrefix("experts").withLayer(0).prefix()); try std.testing.expectEqual(1, layer.withPrefix("experts").count()); try std.testing.expectEqual(0, layer.withPrefix("gate").count()); + + // Every binding of the entry holds the entry itself. + const tensor = layer.createReplicatedTensor("experts.0.down_proj.weight", null); + try std.testing.expectEqual(entry, store.bindingOf(tensor.id).?.load.source.whole().?); } From a1fd1a96d1321c6f670a0e77649605b4b6374ee2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:51:24 +0100 Subject: [PATCH 045/103] zml/io: add Source's rewrite machinery with concatenate and stack Ops that move elements rewrite a source's pieces. A piece is kept flat; an op that addresses axes decomposes it into pieces in the coordinates of the input shape, each dimension moving one axis, cutting it where a dimension would carry from one axis into the next (at row boundaries when the lattice steps regularly, else one piece per step). The op then moves those coordinates and emits each piece flattened in the result's shape, its dimensions outermost first and merged where one continues another. Pieces are allocated in the store's arena, the scratch freed per op. The first ops on it are concatenate and stack, like Tensor's. Along the outermost axis the pieces only shift their origin. Fused QKV, gate_up and expert stacks are interleaves built from splitAxis, concatenate and merge, which keep one piece per checkpoint tensor at checkpoint sizes. Sources of different stores panic. Each op case is checked against Tensor: the same shape-op expression is compiled and run as a program over zml.Tensor, and its bytes and tags are compared with the source's binding lowered and replayed on the host, which also fails on any byte written twice. Co-Authored-By: Claude Opus 5.5 --- zml/io/Source.zig | 512 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 503 insertions(+), 9 deletions(-) diff --git a/zml/io/Source.zig b/zml/io/Source.zig index 1fc93c47..efa1c8a4 100644 --- a/zml/io/Source.zig +++ b/zml/io/Source.zig @@ -1,13 +1,15 @@ //! A tensor the loader assembles from checkpoint bytes by moving them, never -//! by computing: a checkpoint tensor, its axes relabeled or reshaped. The -//! ops are `zml.Tensor`'s shape ops, with the same names, tags and argument -//! conventions. +//! by computing: a checkpoint tensor, or reshapes, concatenations and stacks +//! of checkpoint tensors. The ops are `zml.Tensor`'s shape ops, with the +//! same names, tags and argument conventions. //! //! A source comes from its store (`View.source`) and its pieces live in -//! the store's arena. Nothing is freed before `TensorStore.deinit`, and the -//! store must not move while sources point to it. The ops touch neither the -//! registry nor the bindings. Placement comes last: `withPartitioning` and -//! `as` return a `Placed`, which has no ops and drops its placement with +//! the store's arena, where the ops that rewrite them allocate. Nothing is +//! freed before `TensorStore.deinit`, and the store must not move while +//! sources point to it. The ops touch neither the registry nor the +//! bindings. Sources of different stores do not mix: `concatenate` and +//! `stack` panic on them. Placement comes last: `withPartitioning` and `as` +//! return a `Placed`, which has no ops and drops its placement with //! `unplaced`. A `Placed` is what binds: `Placed.bind` records the model //! tensor the loader fills from it. Sources are built during model init. const std = @import("std"); @@ -167,6 +169,83 @@ fn reshaped(self: Source, new_shape: Shape) Source { return res; } +// Ops that move elements. + +/// Like `Tensor.concatenate`; the sources must share every dimension but +/// `axis_`, and the dtype. +pub fn concatenate(sources: []const Source, axis_: anytype) Source { + stdx.debug.assert(sources.len > 0, "concatenate expects at least one source", .{}); + if (sources.len == 1) return sources[0]; + const first = sources[0]; + const a = first.axis(axis_); + var total: i64 = 0; + for (sources) |s| { + stdx.debug.assert(s.store == first.store, "concatenate expects sources of one store", .{}); + const same = s.dtype() == first.dtype() and s.rank() == first.rank() and for (0..first.rank()) |i| { + if (i != a and s.dim(i) != first.dim(i)) break false; + } else true; + stdx.debug.assert(same, "concatenate along axis {d} expects matching shapes, got {f} and {f}", .{ a, first._shape, s._shape }); + total += s.dim(a); + } + const res_shape = first._shape.setDim(a, total); + const outer_stride = elementStrides(res_shape)[a]; + var ctx: Rewrite = .init(first.store); + defer ctx.deinit(); + var offset: u64 = 0; + for (sources) |s| { + for (s._pieces) |piece| { + if (a == 0) { + // Along the outermost axis, a source's elements follow the previous one's. + var flat: Flat = .fromPiece(piece); + flat.origin += offset * outer_stride; + ctx.emitFlat(flat); + continue; + } + for (ctx.decompose(s._shape, piece)) |*axis_piece| { + axis_piece.origin[a] += offset; + ctx.emit(res_shape, axis_piece.*); + } + } + offset += @intCast(s.dim(a)); + } + return ctx.finish(res_shape); +} + +/// Like `Tensor.stack`; the sources must have the same shape and tags. +pub fn stack(sources: []const Source, axis_: anytype, tag: anytype) Source { + stdx.debug.assert(sources.len > 0, "stack expects at least one source", .{}); + const first = sources[0]; + for (sources[1..]) |s| { + stdx.debug.assert(s.store == first.store, "stack expects sources of one store", .{}); + stdx.debug.assert(first._shape.eqlWithTags(s._shape), "stack expects source shapes to match, got {f} and {f}", .{ first._shape, s._shape }); + } + const a: u3 = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) @intCast(first.rank()) else first.axis(axis_); + const res_shape = first._shape.insertTag(axis_, @intCast(sources.len), tag); + const r = first.rank(); + var ctx: Rewrite = .init(first.store); + defer ctx.deinit(); + for (sources, 0..) |s, i| { + for (s._pieces) |piece| { + if (a == 0) { + var flat: Flat = .fromPiece(piece); + flat.origin += i * first.count(); + ctx.emitFlat(flat); + continue; + } + for (ctx.decompose(s._shape, piece)) |axis_piece| { + var moved = axis_piece; + std.mem.copyBackwards(u64, moved.origin[a + 1 .. r + 1], axis_piece.origin[a..r]); + moved.origin[a] = i; + for (moved.dims.slice()) |*d| { + if (d.axis >= a) d.axis += 1; + } + ctx.emit(res_shape, moved); + } + } + } + return ctx.finish(res_shape); +} + /// This source placed on `mesh_` by `partitioning`: a struct keyed by the /// source's tags (axes it does not name are replicated), or a /// `Sharding.Partitioning`. `.replicated` places it on every device without @@ -283,11 +362,247 @@ fn lowerPiece(allocator: std.mem.Allocator, shape_: Shape, piece: Piece) error{ } }; } +// The machinery. A piece is kept flat; an op that addresses axes rewrites it +// in coordinates of the input shape (`AxisPiece`, every dimension moving one +// axis), moves the coordinates, then flattens it in the result's shape. + +/// A piece while an op rewrites it, its dimensions inline. +const Flat = struct { + tensor: *const safetensors.Tensor, + offset: u64, + origin: u64, + dims: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty, + + fn fromPiece(piece: Piece) Flat { + var flat: Flat = .{ .tensor = piece.tensor, .offset = piece.offset, .origin = piece.origin }; + flat.dims.appendSlice(piece.dims) catch tooManyDims(); + return flat; + } + + /// The same lattice with dimension `k` cut to `n` steps from step `first`. + fn sub(self: Flat, k: usize, first: u64, n: u64) Flat { + var res = self; + const d = self.dims.get(k); + res.origin += first * d.step; + res.offset += first * d.stride; + res.dims.buffer[k].count = n; + return res; + } +}; + +/// A piece in coordinates of a shape: the element at +/// `origin + sum(j[k] * dims[k].step along dims[k].axis)`. +const AxisPiece = struct { + tensor: *const safetensors.Tensor, + offset: u64, + origin: [constants.MAX_RANK]u64 = @splat(0), + dims: stdx.BoundedArray(AxisDim, backend.max_part_dims) = .empty, +}; + +const AxisDim = struct { + count: u64, + axis: u3, + step: u64, + stride: u64, +}; + +/// The scratch of one op and the pieces it makes. +const Rewrite = struct { + store: *TensorStore, + scratch: std.heap.ArenaAllocator, + work: std.ArrayList(Flat) = .empty, + /// The current piece in coordinates of its shape (`decompose`). + axis_pieces: std.ArrayList(AxisPiece) = .empty, + out: std.ArrayList(Piece) = .empty, + + fn init(store: *TensorStore) Rewrite { + return .{ .store = store, .scratch = .init(store.allocator) }; + } + + fn deinit(self: *Rewrite) void { + self.scratch.deinit(); + } + + fn finish(self: *Rewrite, res_shape: Shape) Source { + var s = res_shape; + s._sharding = .replicated; + return .{ + .store = self.store, + ._shape = s, + ._pieces = self.store.arena.allocator().dupe(Piece, self.out.items) catch oom(), + }; + } + + /// `piece` in coordinates of `shape_`, cut where a dimension would carry + /// from one axis into the next; valid until the next call. + fn decompose(self: *Rewrite, shape_: Shape, piece: Piece) []AxisPiece { + const scratch = self.scratch.allocator(); + self.axis_pieces.clearRetainingCapacity(); + self.work.append(scratch, .fromPiece(piece)) catch oom(); + const strides = elementStrides(shape_); + while (self.work.pop()) |flat| { + switch (fitAxes(shape_, strides, flat)) { + .piece => |axis_piece| self.axis_pieces.append(scratch, axis_piece) catch oom(), + .overflow => |overflowing| self.cut(shape_, strides, flat, overflowing), + } + } + return self.axis_pieces.items; + } + + /// Splits `flat`, which carries past the end of axis `a`, into pieces + /// that do not: at the row boundaries of `a` when its longest dimension + /// there steps regularly through the rows, else one piece per step. + fn cut(self: *Rewrite, shape_: Shape, strides: [constants.MAX_RANK]u64, flat: Flat, a: u3) void { + const scratch = self.scratch.allocator(); + const n: u64 = @intCast(shape_.dim(a)); + const c = (flat.origin / strides[a]) % n; + var longest: ?usize = null; + var longest_span: u64 = 0; + var total: u64 = 0; + for (flat.dims.constSlice(), 0..) |d, k| { + if (d.count <= 1 or axisOfStep(shape_.rank(), strides, d.step) != a) continue; + const span = (d.count - 1) * (d.step / strides[a]); + total += span; + if (longest == null or span > longest_span) { + longest = k; + longest_span = span; + } + } + const k = longest.?; + const d = flat.dims.get(k); + const s = d.step / strides[a]; + const rest = total - longest_span; + if (n % s == 0 and c % s + rest < s) { + const per_row = n / s; + var j: u64 = 0; + if (c >= s) { + // A partial first row. + const head = @min((n - c + s - 1) / s, d.count); + self.work.append(scratch, flat.sub(k, 0, head)) catch oom(); + j = head; + } + const rows = (d.count - j) / per_row; + if (rows > 0) { + var body = flat.sub(k, j, per_row); + body.dims.append(.{ .count = rows, .step = d.step * per_row, .stride = d.stride * per_row }) catch tooManyDims(); + self.work.append(scratch, body) catch oom(); + j += rows * per_row; + } + if (j < d.count) self.work.append(scratch, flat.sub(k, j, d.count - j)) catch oom(); + return; + } + // An irregular lattice: one piece per step of that dimension. + for (0..d.count) |step_index| { + var single = flat.sub(k, step_index, 1); + _ = single.dims.orderedRemove(k); + self.work.append(scratch, single) catch oom(); + } + } + + /// Flattens `axis_piece` in `res_shape` into the result. + fn emit(self: *Rewrite, res_shape: Shape, axis_piece: AxisPiece) void { + const strides = elementStrides(res_shape); + var flat: Flat = .{ .tensor = axis_piece.tensor, .offset = axis_piece.offset, .origin = 0 }; + for (0..res_shape.rank()) |b| flat.origin += axis_piece.origin[b] * strides[b]; + for (axis_piece.dims.constSlice()) |d| { + if (d.count <= 1) continue; + flat.dims.append(.{ .count = d.count, .step = d.step * strides[d.axis], .stride = d.stride }) catch tooManyDims(); + } + self.emitFlat(flat); + } + + /// Adds `flat` to the result, its dimensions outermost first and merged + /// where one continues the other. + fn emitFlat(self: *Rewrite, flat: Flat) void { + var sorted: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty; + for (flat.dims.constSlice()) |d| { + if (d.count > 1) sorted.appendAssumeCapacity(d); + } + std.mem.sort(Dim, sorted.slice(), {}, struct { + fn moreOuter(_: void, lhs: Dim, rhs: Dim) bool { + return lhs.step > rhs.step; + } + }.moreOuter); + var merged: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty; + for (sorted.constSlice()) |d| { + if (merged.len != 0) { + const last = &merged.slice()[merged.len - 1]; + if (last.step == d.count * d.step and last.stride == d.count * d.stride) { + last.* = .{ .count = last.count * d.count, .step = d.step, .stride = d.stride }; + continue; + } + } + merged.appendAssumeCapacity(d); + } + const arena = self.store.arena.allocator(); + self.out.append(self.scratch.allocator(), .{ + .tensor = flat.tensor, + .offset = flat.offset, + .origin = flat.origin, + .dims = if (merged.len == 0) &.{} else arena.dupe(Dim, merged.constSlice()) catch oom(), + }) catch oom(); + } +}; + +const AxisFit = union(enum) { + piece: AxisPiece, + /// The axis some coordinate would run past. + overflow: u3, +}; + +/// `flat` in coordinates of `shape_`, if no dimension carries into the next +/// axis: each dimension moves the outermost axis its step is a multiple of. +fn fitAxes(shape_: Shape, strides: [constants.MAX_RANK]u64, flat: Flat) AxisFit { + const r = shape_.rank(); + var axis_piece: AxisPiece = .{ .tensor = flat.tensor, .offset = flat.offset }; + for (0..r) |b| axis_piece.origin[b] = (flat.origin / strides[b]) % @as(u64, @intCast(shape_.dim(b))); + var extent = axis_piece.origin; + for (flat.dims.constSlice()) |d| { + if (d.count <= 1) continue; + const b = axisOfStep(r, strides, d.step); + const step = d.step / strides[b]; + axis_piece.dims.appendAssumeCapacity(.{ .count = d.count, .axis = b, .step = step, .stride = d.stride }); + extent[b] += (d.count - 1) * step; + } + for (0..r) |b| { + if (extent[b] >= @as(u64, @intCast(shape_.dim(b)))) return .{ .overflow = @intCast(b) }; + } + return .{ .piece = axis_piece }; +} + +fn axisOfStep(r: u4, strides: [constants.MAX_RANK]u64, step: u64) u3 { + for (0..r) |b| { + if (step % strides[b] == 0) return @intCast(b); + } + std.debug.panic("a lattice step of {d} elements in a shape of rank {d}", .{ step, r }); +} + +/// Row-major element strides. +fn elementStrides(shape_: Shape) [constants.MAX_RANK]u64 { + var strides: [constants.MAX_RANK]u64 = @splat(0); + var acc: u64 = 1; + var b: usize = shape_.rank(); + while (b > 0) { + b -= 1; + strides[b] = acc; + acc *= @intCast(shape_.dim(b)); + } + return strides; +} + fn oom() noreturn { @panic("OOM"); } -// Tests: the loader's byte placement of a binding, replayed on the host. +fn tooManyDims() noreturn { + std.debug.panic("a Source piece needs more than {d} dimensions", .{backend.max_part_dims}); +} + +// Tests: every op is checked against the same function run as a program +// over `zml.Tensor`, the loader's byte placement replayed on the host. + +const testing = @import("../testing.zig"); +const Buffer = @import("../buffer.zig").Buffer; /// A checkpoint of in-memory tensors (the registry points nowhere) and the /// bytes each holds. @@ -356,17 +671,151 @@ const TestCheckpoint = struct { } }; +fn interleave(comptime T: type, members: []const T, groups: i64) T { + var parts: [4]T = undefined; + for (members, parts[0..members.len]) |member, *part| part.* = member.splitAxis(.row, .{ .group = groups, .r = -1 }); + return T.concatenate(parts[0..members.len], .r).merge(.{ .row = .{ .group, .r } }); +} + +const qkv_inputs = .{ .{ .row = 4, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 } }; +const expert_inputs = .{ .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 } }; + +/// `build(T, x)` builds a tensor from `x` with shape ops only, the same code +/// for `Tensor` and `Source`; `inputs` holds the dims of each `u16` input. +fn expectCase(comptime inputs: anytype, comptime build: anytype) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const n = inputs.len; + var names: [n][]const u8 = undefined; + var name_buffers: [n][16]u8 = undefined; + var shapes: [n]Shape = undefined; + var entries: [n]TestCheckpoint.Entry = undefined; + inline for (inputs, 0..) |dims_, i| { + names[i] = try std.fmt.bufPrint(&name_buffers[i], "input{d}", .{i}); + shapes[i] = Shape.init(dims_, .u16); + entries[i] = .{ .name = names[i], .shape = shapes[i] }; + } + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(allocator, &entries); + defer checkpoint.deinit(); + + var sources: [n]Source = undefined; + for (&sources, names) |*s, input_name| s.* = checkpoint.store.view().source(input_name, null); + const bound = build(Source, &sources).withPartitioning(.replicated, .{}).bind(); + const assembled = try checkpoint.assemble(bound); + defer allocator.free(assembled); + + const platform = testing.env(); + var tensors: [n]Tensor = undefined; + for (&tensors, shapes) |*t, shape_| t.* = .fromShape(shape_); + const Program = struct { + fn forward(x: []const Tensor) Tensor { + return build(Tensor, x); + } + }; + var exe = try platform.compileFn(allocator, io, Program.forward, .{&tensors}, .{}); + defer exe.deinit(); + var buffers: [n]Buffer = undefined; + var created: usize = 0; + defer for (buffers[0..created]) |*buffer| buffer.deinit(); + for (&buffers, shapes, names) |*buffer, shape_, input_name| { + buffer.* = try .fromBytes(io, platform, shape_, checkpoint.bytes.get(input_name).?); + created += 1; + } + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + args.set(.{buffers[0..]}); + try exe.call(io, args, &results, .{ .wait = true }); + var output = results.get(Buffer); + defer output.deinit(); + const expected = try output.toSliceAlloc(allocator, io); + defer expected.free(allocator); + + std.testing.expect(bound.shape().eqlWithTags(output.shape())) catch |err| { + std.log.err("Source gives {f}, Tensor {f}", .{ bound.shape(), output.shape() }); + return err; + }; + std.testing.expectEqualSlices(u8, expected.constData(), assembled) catch |err| { + std.log.err("the loader's bytes differ from the program's", .{}); + return err; + }; +} + +test "Source concatenates rows as Tensor does" { + try expectCase(.{ .{ .row = 3, .col = 4 }, .{ .row = 2, .col = 4 }, .{ .row = 4, .col = 4 } }, struct { + fn build(comptime T: type, x: []const T) T { + return T.concatenate(x, .row); + } + }.build); +} + +test "Source concatenates columns as Tensor does" { + try expectCase(.{ .{ .row = 3, .col = 4 }, .{ .row = 3, .col = 2 } }, struct { + fn build(comptime T: type, x: []const T) T { + return T.concatenate(x, .col); + } + }.build); +} + +test "Source interleaves q, k and v by heads as Tensor does" { + try expectCase(qkv_inputs, struct { + fn build(comptime T: type, x: []const T) T { + return interleave(T, x, 2); + } + }.build); +} + +test "Source interleaves rows as Tensor does" { + try expectCase(.{ .{ .row = 4, .col = 3 }, .{ .row = 4, .col = 3 } }, struct { + fn build(comptime T: type, x: []const T) T { + return interleave(T, x, 4); + } + }.build); +} + +test "Source stacks experts as Tensor does" { + try expectCase(expert_inputs, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; + return T.concatenate(&members, .row); + } + }.build); +} + +test "Source interleaves stacked experts as Tensor does" { + try expectCase(expert_inputs, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; + return T.stack(&members, 2, .proj).merge(.{ .rows = .{ .row, .proj } }); + } + }.build); +} + +test "Source interleaves expert vectors as Tensor does" { + // Per-expert vectors (biases): row r of projection p lands at r * 2 + p. + try expectCase(.{ .{ .row = 3 }, .{ .row = 3 }, .{ .row = 3 }, .{ .row = 3 } }, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; + return T.stack(&members, .last, .proj).merge(.{ .rows = .{ .row, .proj } }); + } + }.build); +} + test "a source binds as its pieces, which the loader lowers to byte parts" { var checkpoint: TestCheckpoint = undefined; try checkpoint.init(std.testing.allocator, &.{ .{ .name = "q", .shape = .init(.{ 4, 3 }, .u16) }, + .{ .name = "k", .shape = .init(.{ 2, 3 }, .u16) }, + .{ .name = "v", .shape = .init(.{ 2, 3 }, .u16) }, }); defer checkpoint.deinit(); var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); defer arena.deinit(); const view = checkpoint.store.view(); const q = view.source("q", .{ .row, .col }); - const entries = [_]*const safetensors.Tensor{view.get("q").?}; + const entries = [_]*const safetensors.Tensor{ view.get("q").?, view.get("k").?, view.get("v").? }; // Relabels keep the whole tensor, one run of it; only `bind` records // anything, and the binding keeps the placed source as it is. @@ -376,6 +825,23 @@ test "a source binds as its pieces, which the loader lowers to byte parts" { try std.testing.expectEqual(relabeled._pieces, bound._pieces); try std.testing.expectEqual(entries[0], bound.whole().?); try std.testing.expectEqualDeep(&[_]backend.LoadPart{.whole(entries[0])}, try bound.lower(arena.allocator())); + // One checkpoint tensor read twice is two pieces of it. + const twice = Source.concatenate(&.{ q, view.source("q", .{ .row, .col }) }, .row).withPartitioning(.replicated, .{}); + try std.testing.expectEqual(null, twice.whole()); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{ + .{ .source = entries[0], .lattice = .{ .dst_offset = 0, .width = 24 } }, + .{ .source = entries[0], .lattice = .{ .dst_offset = 24, .width = 24 } }, + }, try twice.lower(arena.allocator())); + + // q, k and v in two groups: each member is one part of two runs, its + // rows of a group side by side with the others'. + const members = [_]Source{ q, view.source("k", .{ .row, .col }), view.source("v", .{ .row, .col }) }; + const fused = interleave(Source, &members, 2).withPartitioning(.replicated, .{}); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{ + .{ .source = entries[0], .lattice = .{ .src_offset = 0, .dst_offset = 0, .width = 12, .dims = &.{.{ .count = 2, .src_stride = 12, .dst_stride = 24 }} } }, + .{ .source = entries[1], .lattice = .{ .src_offset = 0, .dst_offset = 12, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, + .{ .source = entries[2], .lattice = .{ .src_offset = 0, .dst_offset = 18, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, + }, try fused.lower(arena.allocator())); } test "assembling fails on a byte two pieces write" { @@ -418,3 +884,31 @@ test "a source is placed last, by as or withPartitioning, and binds in its place defer allocator.free(replicated_bytes); try std.testing.expectEqualSlices(u8, replicated_bytes, flat_bytes); } + +test "lattices stay one piece per source at checkpoint sizes" { + // Registered without bytes: nothing is evaluated. + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "memory", .name = "gate", .shape = .init(.{ 14336, 4096 }, .bf16), .offset = 0 }); + try registry.registerTensor(.{ .file_uri = "memory", .name = "up", .shape = .init(.{ 14336, 4096 }, .bf16), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const view = store.view(); + const members = [_]Source{ view.source("gate", .{ .row, .col }), view.source("up", .{ .row, .col }) }; + + // One row per group: two pieces, each 14336 runs of one row. + const finest = interleave(Source, &members, 14336); + try std.testing.expectEqual(@as(usize, 2), finest._pieces.len); + for (try finest.withPartitioning(.replicated, .{}).lower(arena.allocator())) |part| { + try std.testing.expectEqual(@as(u64, 14336), part.lattice.runs()); + try std.testing.expectEqual(@as(u64, 8192), part.lattice.width); + } + // One group per device of four: four runs of 28 MiB each. + const per_shard = interleave(Source, &members, 4); + for (try per_shard.withPartitioning(.replicated, .{}).lower(arena.allocator())) |part| { + try std.testing.expectEqual(@as(u64, 4), part.lattice.runs()); + try std.testing.expectEqual(@as(u64, 3584 * 8192), part.lattice.width); + } +} From c530aa0e6845da67c1dbb783f54c90f75f3d6128 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:53:07 +0100 Subject: [PATCH 046/103] zml/io: add Source slice, split and chunkExact slice is Tensor.slice for a static, contiguous slice: it clips each piece's lattice to the slice's range along the axis, keeping the steps whose whole sub-lattice is inside as one piece and recursing into the partial ones, and shifts the result's origin to the slice's start. A singleton slice drops the axis, and .len takes precedence over the default end as Tensor.Slice resolves it. split and chunkExact are slices; split's result lives in the store's arena. They split fused checkpoint tensors such as gate_up and qkv, and expert slices. Co-Authored-By: Claude Opus 5.5 --- zml/io/Source.zig | 169 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 3 deletions(-) diff --git a/zml/io/Source.zig b/zml/io/Source.zig index efa1c8a4..86b0cd42 100644 --- a/zml/io/Source.zig +++ b/zml/io/Source.zig @@ -1,7 +1,8 @@ //! A tensor the loader assembles from checkpoint bytes by moving them, never -//! by computing: a checkpoint tensor, or reshapes, concatenations and stacks -//! of checkpoint tensors. The ops are `zml.Tensor`'s shape ops, with the -//! same names, tags and argument conventions. +//! by computing: a checkpoint tensor, or slices, reshapes, concatenations +//! and stacks of checkpoint tensors. The ops are `zml.Tensor`'s shape ops, +//! with the same names, tags and argument conventions; slices are static +//! and contiguous. //! //! A source comes from its store (`View.source`) and its pieces live in //! the store's arena, where the ops that rewrite them allocate. Nothing is @@ -171,6 +172,50 @@ fn reshaped(self: Source, new_shape: Shape) Source { // Ops that move elements. +/// Like `Tensor.slice`, for a static, contiguous slice (`step` 1). +pub fn slice(self: Source, axis_: anytype, s: Tensor.Slice) Source { + stdx.debug.assert(s.dyn_start == null, "Source.slice takes static slices, got {f}", .{s}); + stdx.debug.assert(s.step == 1, "Source.slice takes contiguous slices, got {f}", .{s}); + const a = self.axis(axis_); + const range = s.absolute(self.dim(a)); + const lo: u64 = @intCast(range.start); + const hi: u64 = @intCast(range.end); + const res_shape = self._shape.setDim(a, range.end - range.start); + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |axis_piece| ctx.clip(res_shape, axis_piece, a, lo, hi); + } + const sliced = ctx.finish(res_shape); + return if (s.singleton) sliced.reshaped(res_shape.remove(a)) else sliced; +} + +/// Like `Tensor.split`; the result lives in the store's arena. +pub fn split(self: Source, axis_: anytype, split_sizes: []const i64) []Source { + const a = self.axis(axis_); + var total: i64 = 0; + for (split_sizes) |n| total += n; + stdx.debug.assert(total == self.dim(a), "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ total, self.dim(a) }); + const res = self.store.arena.allocator().alloc(Source, split_sizes.len) catch oom(); + var start: i64 = 0; + for (split_sizes, res) |n, *part| { + part.* = self.slice(a, .{ .start = start, .end = start + n }); + start += n; + } + return res; +} + +pub fn chunkExact(self: Source, axis_: anytype, comptime n_chunks: comptime_int) [n_chunks]Source { + const a = self.axis(axis_); + const size = @divExact(self.dim(a), n_chunks); + var chunks: [n_chunks]Source = undefined; + for (&chunks, 0..) |*chunk, i| { + const start = @as(i64, @intCast(i)) * size; + chunk.* = self.slice(a, .{ .start = start, .end = start + size }); + } + return chunks; +} + /// Like `Tensor.concatenate`; the sources must share every dimension but /// `axis_`, and the dtype. pub fn concatenate(sources: []const Source, axis_: anytype) Source { @@ -499,6 +544,56 @@ const Rewrite = struct { } } + /// Emits in `res_shape` the elements of `axis_piece` whose coordinate on + /// `a` is in `[lo, hi)`, at that coordinate minus `lo`. + fn clip(self: *Rewrite, res_shape: Shape, axis_piece: AxisPiece, a: u3, lo: u64, hi: u64) void { + var span: u64 = 0; + var outer: ?usize = null; + for (axis_piece.dims.constSlice(), 0..) |d, k| { + if (d.axis != a or d.count <= 1) continue; + span += (d.count - 1) * d.step; + if (outer == null or d.step > axis_piece.dims.get(outer.?).step) outer = k; + } + const o = axis_piece.origin[a]; + if (o >= lo and o + span < hi) { + var kept = axis_piece; + kept.origin[a] = o - lo; + self.emit(res_shape, kept); + return; + } + if (o + span < lo or o >= hi) return; + // Cut along the outermost dimension on `a`: the steps whose whole + // sub-lattice is inside stay one piece, the partial ones recurse. + const k = outer.?; + const d = axis_piece.dims.get(k); + const inner = span - (d.count - 1) * d.step; + const full_lo: u64 = if (o >= lo) 0 else std.math.divCeil(u64, lo - o, d.step) catch unreachable; + const full_hi: u64 = if (o + inner >= hi) 0 else @min(d.count, (hi - 1 - inner - o) / d.step + 1); + if (full_lo < full_hi) { + var kept = axis_piece; + kept.dims.buffer[k].count = full_hi - full_lo; + kept.origin[a] = o + full_lo * d.step - lo; + kept.offset += full_lo * d.stride; + self.emit(res_shape, kept); + } + const touch_lo: u64 = if (o + inner >= lo) 0 else std.math.divCeil(u64, lo - o - inner, d.step) catch unreachable; + const touch_hi: u64 = @min(d.count, (hi - 1 - o) / d.step + 1); + const partial = [_][2]u64{ + .{ touch_lo, @min(full_lo, touch_hi) }, + .{ @max(full_hi, touch_lo, full_lo), touch_hi }, + }; + for (partial) |range| { + var j = range[0]; + while (j < range[1]) : (j += 1) { + var part = axis_piece; + part.origin[a] = o + j * d.step; + part.offset += j * d.stride; + _ = part.dims.orderedRemove(k); + self.clip(res_shape, part, a, lo, hi); + } + } + } + /// Flattens `axis_piece` in `res_shape` into the result. fn emit(self: *Rewrite, res_shape: Shape, axis_piece: AxisPiece) void { const strides = elementStrides(res_shape); @@ -803,6 +898,71 @@ test "Source interleaves expert vectors as Tensor does" { }.build); } +test "Source slices rows as Tensor does" { + try expectCase(.{.{ .row = 5, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.row, .{ .start = 1, .end = 4 }); + } + }.build); +} + +test "Source slices columns as Tensor does" { + try expectCase(.{.{ .row = 3, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.col, .{ .start = 1, .end = 3 }); + } + }.build); +} + +test "Source slices by length as Tensor does" { + try expectCase(.{.{ .row = 6, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.row, .{ .start = 2, .len = 3 }); + } + }.build); +} + +test "Source slices an interleave as Tensor does" { + try expectCase(qkv_inputs, struct { + fn build(comptime T: type, x: []const T) T { + return interleave(T, x, 2).slice(.row, .{ .start = 3, .end = 7 }); + } + }.build); +} + +test "Source takes a single row as Tensor does" { + try expectCase(.{.{ .row = 5, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.row, .single(2)); + } + }.build); +} + +test "Source splits then interleaves as Tensor does" { + // qwen3_5's in_proj_qkv sliced into q, k and v, interleaved with z. + try expectCase(.{ .{ .row = 6, .col = 3 }, .{ .row = 2, .col = 3 } }, struct { + fn build(comptime T: type, x: []const T) T { + const qkv = x[0]; + const members = [4]T{ + qkv.slice(.row, .{ .start = 0, .end = 2 }), + qkv.slice(.row, .{ .start = 2, .end = 4 }), + qkv.slice(.row, .{ .start = 4, .end = 6 }), + x[1], + }; + return interleave(T, &members, 2); + } + }.build); +} + +test "Source chunks then stacks as Tensor does" { + try expectCase(.{.{ .row = 6, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + const chunks = x[0].chunkExact(.row, 3); + return T.stack(&.{ chunks[2], chunks[0] }, .last, .pair).insertAxes(0, .{.one}).squeeze(0); + } + }.build); +} + test "a source binds as its pieces, which the loader lowers to byte parts" { var checkpoint: TestCheckpoint = undefined; try checkpoint.init(std.testing.allocator, &.{ @@ -911,4 +1071,7 @@ test "lattices stay one piece per source at checkpoint sizes" { try std.testing.expectEqual(@as(u64, 4), part.lattice.runs()); try std.testing.expectEqual(@as(u64, 3584 * 8192), part.lattice.width); } + // Slicing through the interleave cuts at most a few pieces. + const sliced = finest.slice(.row, .{ .start = 1001, .end = 20001 }); + try std.testing.expect(sliced._pieces.len <= 6); } From d899035ddd782a5e9532b16fb18eb7c966c2003c Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:54:10 +0100 Subject: [PATCH 047/103] zml/io: add Source.transpose transpose is Tensor.transpose, and no axes reverses them. When the axes that move all hold one element and the others keep their order, the row-major order is unchanged and the transpose is a reshape that leaves the pieces alone. Otherwise each piece's origin and dimensions are permuted, and a transposed tensor lowers to strided parts of single elements. Placing a transposed source with as keeps its element order. Co-Authored-By: Claude Opus 5.5 --- zml/io/Source.zig | 80 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/zml/io/Source.zig b/zml/io/Source.zig index 86b0cd42..f441de9b 100644 --- a/zml/io/Source.zig +++ b/zml/io/Source.zig @@ -1,8 +1,8 @@ //! A tensor the loader assembles from checkpoint bytes by moving them, never -//! by computing: a checkpoint tensor, or slices, reshapes, concatenations -//! and stacks of checkpoint tensors. The ops are `zml.Tensor`'s shape ops, -//! with the same names, tags and argument conventions; slices are static -//! and contiguous. +//! by computing: a checkpoint tensor, or slices, reshapes, transposes, +//! concatenations and stacks of checkpoint tensors. The ops are +//! `zml.Tensor`'s shape ops, with the same names, tags and argument +//! conventions; slices are static and contiguous. //! //! A source comes from its store (`View.source`) and its pieces live in //! the store's arena, where the ops that rewrite them allocate. Nothing is @@ -216,6 +216,44 @@ pub fn chunkExact(self: Source, axis_: anytype, comptime n_chunks: comptime_int) return chunks; } +/// Like `Tensor.transpose`: no axes reverses them. +pub fn transpose(self: Source, axes_: anytype) Source { + const given = self._shape.axes(axes_); + const r = self.rank(); + var permutation: [constants.MAX_RANK]u3 = undefined; + if (given.len == 0) { + for (0..r) |i| permutation[i] = @intCast(r - 1 - i); + } else { + stdx.debug.assert(given.len == r, "transpose expects {d} axes for {f}, got {d}", .{ r, self._shape, given.len }); + @memcpy(permutation[0..r], given.constSlice()); + } + var as_i64: [constants.MAX_RANK]i64 = undefined; + for (permutation[0..r], as_i64[0..r]) |p, *q| q.* = p; + const res_shape = self._shape.transpose(as_i64[0..r]); + // Axes of one element move nothing: when the others keep their order, + // the row-major order is unchanged, so are the pieces. + var previous: ?u3 = null; + const moves = for (permutation[0..r]) |p| { + if (self.dim(p) == 1) continue; + if (previous) |q| if (p < q) break true; + previous = p; + } else false; + if (!moves) return self.reshaped(res_shape); + var inverse: [constants.MAX_RANK]u3 = undefined; + for (permutation[0..r], 0..) |p, i| inverse[p] = @intCast(i); + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |axis_piece| { + var moved = axis_piece; + for (0..r) |i| moved.origin[i] = axis_piece.origin[permutation[i]]; + for (moved.dims.slice()) |*d| d.axis = inverse[d.axis]; + ctx.emit(res_shape, moved); + } + } + return ctx.finish(res_shape); +} + /// Like `Tensor.concatenate`; the sources must share every dimension but /// `axis_`, and the dtype. pub fn concatenate(sources: []const Source, axis_: anytype) Source { @@ -898,6 +936,23 @@ test "Source interleaves expert vectors as Tensor does" { }.build); } +test "Source transposes as Tensor does" { + try expectCase(.{.{ .row = 3, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].transpose(.{ .col, .row }); + } + }.build); +} + +test "Source interleaves rows of three as Tensor does" { + // lfm2's in_proj: rows [B; C; x] become b0 c0 x0 b1 c1 x1. + try expectCase(.{.{ .row = 6, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].splitAxis(.row, .{ .m = 3, .c = 2 }).transpose(.{ .c, .m, .col }).merge(.{ .row = .{ .c, .m } }); + } + }.build); +} + test "Source slices rows as Tensor does" { try expectCase(.{.{ .row = 5, .col = 4 }}, struct { fn build(comptime T: type, x: []const T) T { @@ -1002,6 +1057,15 @@ test "a source binds as its pieces, which the loader lowers to byte parts" { .{ .source = entries[1], .lattice = .{ .src_offset = 0, .dst_offset = 12, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, .{ .source = entries[2], .lattice = .{ .src_offset = 0, .dst_offset = 18, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, }, try fused.lower(arena.allocator())); + + // A transpose is one part of single elements. + const transposed = q.transpose(.{}).withPartitioning(.replicated, .{}); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{ + .{ .source = entries[0], .lattice = .{ .src_offset = 0, .dst_offset = 0, .width = 2, .dims = &.{ + .{ .count = 3, .src_stride = 2, .dst_stride = 8 }, + .{ .count = 4, .src_stride = 6, .dst_stride = 2 }, + } } }, + }, try transposed.lower(arena.allocator())); } test "assembling fails on a byte two pieces write" { @@ -1021,17 +1085,17 @@ test "a source is placed last, by as or withPartitioning, and binds in its place try checkpoint.init(allocator, &.{.{ .name = "w", .shape = .init(.{ 4, 6 }, .u16) }}); defer checkpoint.deinit(); const mesh: Sharding.Mesh = .initTest(.tp, .{.model}); - const w = checkpoint.store.view().source("w", .{ .row, .col }); + const transposed = checkpoint.store.view().source("w", .{ .row, .col }).transpose(.{}); // `as` takes a shape built elsewhere, here of another rank: its // dimensions, tags and sharding, for the same elements. const placed = Shape.init(.{ .flat = 24 }, .u16).withPartitioning(&mesh, .{ .flat = .model }); - const placed_source = w.as(placed); + const placed_source = transposed.as(placed); const flat = placed_source.bind(); try std.testing.expect(flat.shape().eqlWithTagsAndPartitioning(placed)); try std.testing.expect(checkpoint.store.bindingOf(flat.id).?.load.source.shape.eqlWithTagsAndPartitioning(placed)); - const replicated = w.withPartitioning(.replicated, .{}).bind(); - try std.testing.expect(replicated.shape().eqlWithTags(w.shape())); + const replicated = transposed.withPartitioning(.replicated, .{}).bind(); + try std.testing.expect(replicated.shape().eqlWithTags(transposed.shape())); try std.testing.expectEqual(null, replicated.shape()._sharding.mesh); // `unplaced` keeps the placed dimensions and tags, without the sharding. const dropped = placed_source.unplaced(); From fd268c912b97b5694bc9f2d4b0d4ad5e3ba3d32a Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:55:09 +0100 Subject: [PATCH 048/103] zml/io: add Source.pad pad is Tensor.pad with zero as the only padding value: low, high and interior padding are nonnegative, so it never crops. Each piece's origin and steps move by the padding along every axis, and the elements no piece covers are the zeros. Rows appended after the last one move no element, so that case only grows the shape. It replaces padding a table by hand over the old store's writers. Co-Authored-By: Claude Opus 5.5 --- zml/io/Source.zig | 59 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/zml/io/Source.zig b/zml/io/Source.zig index f441de9b..f1877273 100644 --- a/zml/io/Source.zig +++ b/zml/io/Source.zig @@ -1,8 +1,9 @@ //! A tensor the loader assembles from checkpoint bytes by moving them, never //! by computing: a checkpoint tensor, or slices, reshapes, transposes, -//! concatenations and stacks of checkpoint tensors. The ops are +//! paddings, concatenations and stacks of checkpoint tensors. The ops are //! `zml.Tensor`'s shape ops, with the same names, tags and argument -//! conventions; slices are static and contiguous. +//! conventions; slices are static and contiguous, and padding is zeros +//! only. //! //! A source comes from its store (`View.source`) and its pieces live in //! the store's arena, where the ops that rewrite them allocate. Nothing is @@ -57,6 +58,9 @@ const Dim = struct { stride: u64, }; +/// `pad`'s per-axis options, as for `Tensor.pad`. +pub const Pad = Tensor.Pad; + /// The checkpoint tensor `subkey` of `view`, with `tags` (null keeps the /// checkpoint's); null when the checkpoint has no such tensor. pub fn init(view: TensorStore.View, subkey: []const u8, tags: ?[]const Shape.Tag) ?Source { @@ -254,6 +258,34 @@ pub fn transpose(self: Source, axes_: anytype) Source { return ctx.finish(res_shape); } +/// Like `Tensor.pad`, with zero as the only padding value. +pub fn pad(self: Source, padding_value: anytype, paddings: anytype) Source { + stdx.debug.assert(isZero(padding_value), "Source.pad pads with zeros, got {any}", .{padding_value}); + const options = self._shape.parseAxesOptions(Tensor.Pad, paddings, .{}); + var res_shape = self._shape; + var moves = false; + for (options.constSlice(), 0..) |p, a| { + stdx.debug.assert(p.low >= 0 and p.high >= 0 and p.interior >= 0, "Source.pad cannot crop, got {any} on axis {d}", .{ p, a }); + const d = self.dim(a); + res_shape._dims.set(a, p.low + d + @max(d - 1, 0) * p.interior + p.high); + if (p.low != 0 or p.interior != 0 or (a != 0 and p.high != 0)) moves = true; + } + // Rows appended after the last one move no element. + if (!moves) return .{ .store = self.store, ._shape = res_shape, ._pieces = self._pieces }; + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |*axis_piece| { + for (options.constSlice(), 0..) |p, a| { + axis_piece.origin[a] = @as(u64, @intCast(p.low)) + axis_piece.origin[a] * @as(u64, @intCast(p.interior + 1)); + } + for (axis_piece.dims.slice()) |*d| d.step *= @intCast(options.get(d.axis).interior + 1); + ctx.emit(res_shape, axis_piece.*); + } + } + return ctx.finish(res_shape); +} + /// Like `Tensor.concatenate`; the sources must share every dimension but /// `axis_`, and the dtype. pub fn concatenate(sources: []const Source, axis_: anytype) Source { @@ -723,6 +755,13 @@ fn elementStrides(shape_: Shape) [constants.MAX_RANK]u64 { return strides; } +fn isZero(value: anytype) bool { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_int, .int, .comptime_float, .float => value == 0, + else => false, + }; +} + fn oom() noreturn { @panic("OOM"); } @@ -993,6 +1032,22 @@ test "Source takes a single row as Tensor does" { }.build); } +test "Source pads as Tensor does" { + try expectCase(.{.{ .row = 4, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].pad(0, .{ .row = Tensor.Pad{ .high = 4 }, .col = Tensor.Pad{ .low = 1, .interior = 1 } }); + } + }.build); +} + +test "Source pads rows as Tensor does" { + try expectCase(.{.{ .row = 5, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].pad(0, .{ .row = Tensor.Pad{ .high = 3 } }); + } + }.build); +} + test "Source splits then interleaves as Tensor does" { // qwen3_5's in_proj_qkv sliced into q, k and v, interleaved with z. try expectCase(.{ .{ .row = 6, .col = 3 }, .{ .row = 2, .col = 3 } }, struct { From 84365eb681066e4c5a03d78977a642a77c2f5497 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:56:56 +0100 Subject: [PATCH 049/103] zml/io: add Source.broad broad is Tensor.broad: the elements repeat along the axes the target shape adds or grows from 1, as dimensions whose checkpoint stride is zero. The loader reads a few large repetitions once each, and copies many small ones from one read of the source. Shared scales are bound this way. Co-Authored-By: Claude Opus 5.5 --- zml/io/Source.zig | 66 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/zml/io/Source.zig b/zml/io/Source.zig index f1877273..f12fa835 100644 --- a/zml/io/Source.zig +++ b/zml/io/Source.zig @@ -1,9 +1,9 @@ //! A tensor the loader assembles from checkpoint bytes by moving them, never //! by computing: a checkpoint tensor, or slices, reshapes, transposes, -//! paddings, concatenations and stacks of checkpoint tensors. The ops are -//! `zml.Tensor`'s shape ops, with the same names, tags and argument -//! conventions; slices are static and contiguous, and padding is zeros -//! only. +//! broadcasts, paddings, concatenations and stacks of checkpoint tensors. +//! The ops are `zml.Tensor`'s shape ops, with the same names, tags and +//! argument conventions; slices are static and contiguous, and padding is +//! zeros only. //! //! A source comes from its store (`View.source`) and its pieces live in //! the store's arena, where the ops that rewrite them allocate. Nothing is @@ -286,6 +286,46 @@ pub fn pad(self: Source, padding_value: anytype, paddings: anytype) Source { return ctx.finish(res_shape); } +/// Like `Tensor.broad`: repeats the elements along the axes `other` adds or +/// grows from 1. The loader reads a few large repetitions once each, and +/// copies many small ones from one read of the source per job. +pub fn broad(self: Source, other: Shape) Source { + stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self._shape, other }); + if (std.mem.eql(i64, self.dims(), other.dims())) return self; + const r = self.rank(); + var axes_map: [constants.MAX_RANK]u3 = undefined; + if (r == 0 or r == other.rank()) { + for (0..r) |i| axes_map[i] = @intCast(i); + } else { + for (self._shape.tags(), 0..) |t, i| axes_map[i] = other.axis(t); + } + const res_shape = other.withDtype(self.dtype()); + // The axes the elements repeat along: new ones and those grown from 1. + var repeated: [constants.MAX_RANK]bool = @splat(true); + for (0..r) |i| { + if (self.dim(i) != 1) repeated[axes_map[i]] = false; + } + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |axis_piece| { + var moved: AxisPiece = .{ .tensor = axis_piece.tensor, .offset = axis_piece.offset }; + for (0..r) |i| moved.origin[axes_map[i]] = axis_piece.origin[i]; + for (axis_piece.dims.constSlice()) |d| { + var mapped = d; + mapped.axis = axes_map[d.axis]; + moved.dims.append(mapped) catch tooManyDims(); + } + for (0..other.rank()) |b| { + if (!repeated[b] or other.dim(b) == 1) continue; + moved.dims.append(.{ .count = @intCast(other.dim(b)), .axis = @intCast(b), .step = 1, .stride = 0 }) catch tooManyDims(); + } + ctx.emit(res_shape, moved); + } + } + return ctx.finish(res_shape); +} + /// Like `Tensor.concatenate`; the sources must share every dimension but /// `axis_`, and the dtype. pub fn concatenate(sources: []const Source, axis_: anytype) Source { @@ -1048,6 +1088,24 @@ test "Source pads rows as Tensor does" { }.build); } +test "Source broadcasts a scalar as Tensor does" { + try expectCase(.{.{}}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].broad(Shape.init(.{ .row = 3, .col = 2 }, .u16)); + } + }.build); +} + +test "Source broadcasts a column as Tensor does" { + // A per-tensor scale repeated over each member's rows. + try expectCase(.{ .{ .row = 1, .col = 1 }, .{ .row = 1, .col = 1 } }, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ x[0].broad(Shape.init(.{ .row = 4, .col = 1 }, .u16)), x[1].broad(Shape.init(.{ .row = 2, .col = 1 }, .u16)) }; + return interleave(T, &members, 2); + } + }.build); +} + test "Source splits then interleaves as Tensor does" { // qwen3_5's in_proj_qkv sliced into q, k and v, interleaved with z. try expectCase(.{ .{ .row = 6, .col = 3 }, .{ .row = 2, .col = 3 } }, struct { From dcafea2d067d19f19d98926faaca474e6fdea723 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:57:54 +0100 Subject: [PATCH 050/103] zml/io: add Source.bitCast bitCast reinterprets bits like Tensor.bitCast. To a narrower type, each element becomes the elements of a new last axis .bitcast. To a wider type, the last axis must hold one element of the new type, and its bits must be one contiguous run in the checkpoint, which panics otherwise. Sub-byte types that land inside a byte fail at lowering with SubBytePosition. FP8 and NVFP4 block scales, including the 128x4 swizzle, are laid out as sources this way: the swizzle case pads, splits, transposes and merges, checked against Tensor. Co-Authored-By: Claude Opus 5.5 --- zml/io/Source.zig | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/zml/io/Source.zig b/zml/io/Source.zig index f12fa835..6f730515 100644 --- a/zml/io/Source.zig +++ b/zml/io/Source.zig @@ -174,6 +174,49 @@ fn reshaped(self: Source, new_shape: Shape) Source { return res; } +/// Reinterprets the bits as `dt`, like `Tensor.bitCast`: to a narrower type, +/// each element becomes the elements of a new last axis `.bitcast`; to a +/// wider one, the last axis must hold one element of `dt`. +pub fn bitCast(self: Source, dt: DataType) Source { + const src_bits: u64 = self.dtype().bitSizeOf(); + const tgt_bits: u64 = dt.bitSizeOf(); + if (src_bits == tgt_bits) return .{ .store = self.store, ._shape = self._shape.withDtype(dt), ._pieces = self._pieces }; + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + if (src_bits > tgt_bits) { + const ratio = std.math.divExact(u64, src_bits, tgt_bits) catch std.debug.panic("bitCast from {t} to {t}: bit sizes are not multiples", .{ self.dtype(), dt }); + const res_shape = self._shape.append(.{ .bitcast = @as(i64, @intCast(ratio)) }).withDtype(dt); + for (self._pieces) |piece| { + var flat: Flat = .fromPiece(piece); + flat.origin *= ratio; + for (flat.dims.slice()) |*d| d.step *= ratio; + flat.dims.append(.{ .count = ratio, .step = 1, .stride = tgt_bits }) catch tooManyDims(); + ctx.emitFlat(flat); + } + return ctx.finish(res_shape); + } + const ratio = std.math.divExact(u64, tgt_bits, src_bits) catch std.debug.panic("bitCast from {t} to {t}: bit sizes are not multiples", .{ self.dtype(), dt }); + const last: u3 = @intCast(self.rank() - 1); + stdx.debug.assert(self.dim(last) == @as(i64, @intCast(ratio)), "bitCast from {f} to {t} expects a last axis of {d} elements", .{ self._shape, dt, ratio }); + const res_shape = self._shape.remove(last).withDtype(dt); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |*axis_piece| { + // The elements merged into one must be one contiguous run of bits. + const inner: ?usize = for (axis_piece.dims.constSlice(), 0..) |d, k| { + if (d.axis == last) break k; + } else null; + const whole = axis_piece.origin[last] == 0 and inner != null and blk: { + const d = axis_piece.dims.get(inner.?); + break :blk d.count == ratio and d.step == 1 and d.stride == src_bits; + }; + if (!whole) std.debug.panic("bitCast of {f} to {t}: an element of {t} would gather bits that are not contiguous in the checkpoint", .{ self._shape, dt, dt }); + _ = axis_piece.dims.orderedRemove(inner.?); + ctx.emit(res_shape, axis_piece.*); + } + } + return ctx.finish(res_shape); +} + // Ops that move elements. /// Like `Tensor.slice`, for a static, contiguous slice (`step` 1). @@ -1106,6 +1149,36 @@ test "Source broadcasts a column as Tensor does" { }.build); } +test "Source bitcasts to a narrower type as Tensor does" { + try expectCase(.{.{ .row = 3, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].bitCast(.u8).merge(.{ .col = .{ .col, .bitcast } }); + } + }.build); +} + +test "Source bitcasts to a wider type as Tensor does" { + try expectCase(.{.{ .row = 2, .col = 6 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].bitCast(.u8).splitAxis(.col, .{ .col = 3, .pair = 2 }).merge(.{ .pair = .{ .pair, .bitcast } }).bitCast(.u32); + } + }.build); +} + +test "Source swizzles block scales as Tensor does" { + // The 128x4 block-scale layout, at 8x4: rows padded to 8 and tiled + // by 2x4, columns padded to 4. + try expectCase(.{.{ .e = 2, .m = 7, .k = 6 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].pad(0, .{ .m = Tensor.Pad{ .high = 1 }, .k = Tensor.Pad{ .high = 2 } }) + .splitAxis(.m, .{ .mo = -1, .m4 = 2, .mi = 4 }) + .splitAxis(.k, .{ .ko = -1, .ki = 4 }) + .transpose(.{ .e, .mo, .ko, .mi, .m4, .ki }) + .merge(.{ .flat = .{ .mo, .ko, .mi, .m4, .ki } }); + } + }.build); +} + test "Source splits then interleaves as Tensor does" { // qwen3_5's in_proj_qkv sliced into q, k and v, interleaved with z. try expectCase(.{ .{ .row = 6, .col = 3 }, .{ .row = 2, .col = 3 } }, struct { From 7b3c68d30ec475fa8378c5b1ef13e82c7971e7fe Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:59:52 +0100 Subject: [PATCH 051/103] zml/io: bind programs over Sources and set memory after binding A tensor can now be computed while loading instead of filled from a source. TensorStore.Program is a program value: a plain-data struct, checked at compile time to hold no pointer, whose type declares forward(self, output, inputs). Program.init copies the value into the store's arena behind a per-type vtable; as reads it back and name, the type's name, names the compiled module. Binding gains its compute arm, a program and its placed inputs in order. Compute.sameExecutable tells whether two bindings compile to one executable: programs of one type with equal values, over inputs and outputs whose shapes are equal with their tags and partitioning. Compute.compile compiles the program on the store's meshes. Source.bindProgram binds one over placed sources of one store. The loader that compiles and runs these programs comes later. View.setMemory is the one way to change where the loader delivers a bound tensor after binding; it panics on a program's output, which lands in default device memory. View.hostMappedBytes sums, over the whole store, one device's shard of every tensor delivered host_mapped: what XLA counts against device memory for those views, which a model passes back through Compiler.Options.extra_device_memory_bytes, whose doc now points to it. A tensor whose mesh does not divide its dimensions is IncompatibleSharding, logged. Co-Authored-By: Claude Opus 5.5 --- zml/Compiler.zig | 3 +- zml/io/Source.zig | 42 ++++-- zml/io/TensorStore.zig | 296 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 314 insertions(+), 27 deletions(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 64dab3ae..eebf0be5 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -84,7 +84,8 @@ pub const Options = struct { /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. /// XLA then counts them against device memory, leaves no budget for temporary buffers and /// rematerializes aggressively. Callers set this to the per-device size of such buffers to - /// compensate. Remove once XLA properly supports host-pinned inputs. + /// compensate: `TensorStore.View.hostMappedBytes` for those the loader delivers + /// `.host_mapped`. Remove once XLA properly supports host-pinned inputs. extra_device_memory_bytes: u64 = 0, // Debugging options program_name: []const u8 = "zml", diff --git a/zml/io/Source.zig b/zml/io/Source.zig index 6f730515..ea90111a 100644 --- a/zml/io/Source.zig +++ b/zml/io/Source.zig @@ -9,11 +9,13 @@ //! the store's arena, where the ops that rewrite them allocate. Nothing is //! freed before `TensorStore.deinit`, and the store must not move while //! sources point to it. The ops touch neither the registry nor the -//! bindings. Sources of different stores do not mix: `concatenate` and -//! `stack` panic on them. Placement comes last: `withPartitioning` and `as` -//! return a `Placed`, which has no ops and drops its placement with -//! `unplaced`. A `Placed` is what binds: `Placed.bind` records the model -//! tensor the loader fills from it. Sources are built during model init. +//! bindings. Sources of different stores do not mix: `concatenate`, +//! `stack` and `bindProgram` panic on them. Placement comes last: +//! `withPartitioning` and `as` return a `Placed`, which has no ops and +//! drops its placement with `unplaced`. A `Placed` is what binds: +//! `Placed.bind` records the model tensor the loader fills from it, +//! `bindProgram` one a program computes from several. Sources are built +//! during model init. const std = @import("std"); const stdx = @import("stdx"); @@ -448,7 +450,8 @@ pub fn stack(sources: []const Source, axis_: anytype, tag: anytype) Source { /// source's tags (axes it does not name are replicated), or a /// `Sharding.Partitioning`. `.replicated` places it on every device without /// a mesh, and takes only `.{}`. Placement comes last: a `Placed` only -/// binds (`Placed.bind`) or drops its placement (`Placed.unplaced`). +/// binds (`Placed.bind`, `bindProgram`) or drops its placement +/// (`Placed.unplaced`). pub fn withPartitioning(self: Source, comptime mesh_: @EnumLiteral(), partitioning: anytype) Placed { var placed = self._shape; if (mesh_ == .replicated) { @@ -470,6 +473,25 @@ pub fn as(self: Source, placed: Shape) Placed { return .{ .store = self.store, .shape = placed, ._pieces = self._pieces }; } +/// Makes a model tensor of `output` that `program` computes while loading, +/// `program.forward(output, inputs)` over the placed sources `inputs` in this +/// order (`TensorStore.Program.init`). The loader lands each input as it +/// lands a bound source, placed by its shape, in default device memory; +/// compiles the program once for every tensor of an equal program over the +/// same placed shapes; runs it, and frees the inputs. `forward` returns a +/// tensor of `output`, placed alike, in default device memory. +pub fn bindProgram(inputs: []const Placed, output: Shape, program: anytype) Tensor { + stdx.debug.assert(inputs.len != 0, "Source.bindProgram: a program computing {f} needs an input", .{output}); + const store = inputs[0].store; + for (inputs) |input| { + stdx.debug.assert(input.store == store, "Source.bindProgram: the inputs of {f} come from different stores", .{output}); + } + const arena = store.arena.allocator(); + const kept = arena.dupe(Placed, inputs) catch oom(); + const erased = TensorStore.Program.init(arena, program) catch oom(); + return bindNew(store, output, .{ .compute = .{ .program = erased, .inputs = kept } }); +} + /// A source with its placement (`withPartitioning`, `as`): the only thing /// that binds, and what a binding keeps (`TensorStore.Binding`). pub const Placed = struct { @@ -480,8 +502,8 @@ pub const Placed = struct { _pieces: []const Piece, /// Makes the model tensor the loader fills from this source, in default - /// memory. The binding keeps the source as it is; the loader lowers it - /// to bytes (`lower`). + /// memory unless `View.setMemory` names another. The binding keeps the + /// source as it is; the loader lowers it to bytes (`lower`). pub fn bind(self: Placed) Tensor { return bindNew(self.store, self.shape, .{ .load = .{ .source = self } }); } @@ -522,8 +544,8 @@ pub const Placed = struct { }; /// A new tensor of `shape_`, bound to `binding`: the one maker of -/// `store`'s bindings (`Placed.bind`), whose pieces are disjoint by -/// construction: the loader relies on that and does not check. +/// `store`'s bindings (`Placed.bind`, `bindProgram`), whose pieces are +/// disjoint by construction: the loader relies on that and does not check. fn bindNew(store: *TensorStore, shape_: Shape, binding: TensorStore.Binding) Tensor { const tensor: Tensor = .fromShape(shape_); store.id_to_sources.putNoClobber(store.allocator, tensor.id, binding) catch oom(); diff --git a/zml/io/TensorStore.zig b/zml/io/TensorStore.zig index 849729b3..bc40c775 100644 --- a/zml/io/TensorStore.zig +++ b/zml/io/TensorStore.zig @@ -4,27 +4,32 @@ //! `zml.io.TensorStore` (io.zig), which borrows its registry, so production //! changes go there and the new loader's here. //! -//! The store owns those entries, the bindings, and every source (its -//! arena). Views and sources point to it: once `view` was called it must -//! not move, and it must outlive them. It borrows the meshes of -//! `Options.meshes`, which belong to the platform. Binding cannot fail: a -//! missing key panics with the key. +//! The store owns those entries, the bindings, and every source and program +//! value (its arena). Views and sources point to it: once `view` was +//! called it must not move, and it must outlive them. It borrows the meshes +//! of `Options.meshes`, which belong to the platform. Binding cannot fail: +//! a missing key panics with the key. const std = @import("std"); const stdx = @import("stdx"); +const Compiler = @import("../Compiler.zig"); +const Exe = @import("../exe.zig").Exe; const MemorySpace = @import("../platform.zig").Memory.Kind; +const Platform = @import("../platform.zig").Platform; const Quantization = @import("../quantization.zig").Quantization; const safetensors = @import("../safetensors.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; +const load_log = @import("log.zig").load; +const testing = @import("../testing.zig"); const TensorStore = @This(); -/// Where the loader delivers a loaded tensor, on every device of its mesh. -/// The destination is allocated once per device whatever its pieces; the -/// host kinds take no device memory. +/// Where the loader delivers a loaded tensor, on every device of its mesh +/// (`View.setMemory`). The destination is allocated once per device +/// whatever its pieces; the host kinds take no device memory. pub const Memory = enum { /// Device memory. default, @@ -35,7 +40,7 @@ pub const Memory = enum { /// buffer is a device-kind view of it (`createViewOfDeviceBuffer`), /// and the memory goes with the last view. For a table too large for /// device memory, of which a step reads a few rows. XLA counts the - /// views against device memory. + /// views against device memory (`View.hostMappedBytes`). host_mapped, /// Whether the bytes live in host memory, written by the CPU. @@ -55,18 +60,155 @@ pub const Memory = enum { } }; -/// What the loader reads of a bound tensor (`bindingOf`): the source it -/// fills the tensor from. +/// What the loader reads of a bound tensor (`bindingOf`): the source +/// it fills the tensor from, or the program that computes it. pub const Binding = union(enum) { /// The loader fills the tensor from `source` (`Source.Placed.bind`), /// every byte no piece covers zero. load: Load, + /// A program computes the tensor while loading (`Source.bindProgram`). + /// Its output lands in default device memory. + compute: Compute, pub const Load = struct { source: Source.Placed, - /// Where the loader delivers this tensor. + /// Where the loader delivers this tensor (`View.setMemory`). memory: Memory = .default, }; + + pub const Compute = struct { + program: Program, + /// The program's inputs in order, each landed in default device + /// memory, placed by its shape. + inputs: []const Source.Placed, + + /// Whether this computing `output` and `other` computing + /// `other_output` compile to the same executable: equal programs + /// over the same shapes placed alike. + pub fn sameExecutable(self: Compute, output: Shape, other: Compute, other_output: Shape) bool { + if (!self.program.eql(other.program)) return false; + if (!output.eqlWithTagsAndPartitioning(other_output) or self.inputs.len != other.inputs.len) return false; + for (self.inputs, other.inputs) |a, b| { + if (!a.shape.eqlWithTagsAndPartitioning(b.shape)) return false; + } + return true; + } + + /// Compiles the program computing a tensor of `output` from inputs + /// of the shapes it binds, its module named after the program. + pub fn compile( + self: Compute, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + meshes: []const *const Sharding.Mesh, + output: Shape, + ) Compiler.Error!Exe { + const inputs = try allocator.alloc(Tensor, self.inputs.len); + defer allocator.free(inputs); + for (inputs, self.inputs) |*input, source| input.* = .fromShape(source.shape); + const opts: Compiler.Options = .{ .meshes = meshes, .program_name = self.program.name() }; + return self.program.vtable.compile(self.program.value, allocator, io, platform, opts, output, inputs); + } + }; +}; + +/// A program the loader compiles and runs to compute a tensor while loading +/// (`Binding.compute`): `forward(output, inputs)` of the program's value. +/// Model code makes one through `Source.bindProgram`, and reads it back +/// with `as`. +pub const Program = struct { + /// The program's value, in the store's arena. + value: *const anyopaque, + vtable: *const VTable, + + const VTable = struct { + name: []const u8, + compile: *const fn ( + value: *const anyopaque, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Compiler.Options, + output: Shape, + inputs: []const Tensor, + ) Compiler.Error!Exe, + eql: *const fn (a: *const anyopaque, b: *const anyopaque) bool, + }; + + /// `program`, its value copied into `arena`. Its type declares + /// `pub fn forward(self, output: Shape, inputs: []const Tensor) Tensor` + /// and is plain data, without pointers, so equal values are found by + /// value and compile once. + pub fn init(arena: std.mem.Allocator, program: anytype) std.mem.Allocator.Error!Program { + const T = @TypeOf(program); + comptime { + if (!isPlainData(T)) @compileError("TensorStore.Program: " ++ @typeName(T) ++ " must be plain data, without pointers"); + } + const copy = try arena.create(T); + copy.* = program; + return .{ .value = copy, .vtable = &Erased(T).vtable }; + } + + /// The program's value, if it is a `T`. + pub fn as(self: Program, comptime T: type) ?T { + if (self.vtable != &Erased(T).vtable) return null; + return Erased(T).typed(self.value); + } + + /// The program's type name, which names its compiled module in traces, + /// dumps and logs. + pub fn name(self: Program) []const u8 { + return self.vtable.name; + } + + /// Whether `a` and `b` are programs of one type with equal values. + fn eql(a: Program, b: Program) bool { + return a.vtable == b.vtable and a.vtable.eql(a.value, b.value); + } + + /// The vtable of programs of type `T`, one per type. + fn Erased(comptime T: type) type { + return struct { + const vtable: VTable = .{ .name = @typeName(T), .compile = compileValue, .eql = eqlValues }; + + fn typed(erased: *const anyopaque) T { + return @as(*const T, @ptrCast(@alignCast(erased))).*; + } + + fn compileValue( + erased: *const anyopaque, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Compiler.Options, + output: Shape, + inputs: []const Tensor, + ) Compiler.Error!Exe { + return Compiler.compile(allocator, io, platform, T.forward, .{ typed(erased), output, inputs }, opts); + } + + fn eqlValues(a: *const anyopaque, b: *const anyopaque) bool { + return std.meta.eql(typed(a), typed(b)); + } + }; + } + + /// Values compared by value: no pointer anywhere inside. + fn isPlainData(comptime T: type) bool { + return switch (@typeInfo(T)) { + .void, .bool, .int, .float, .@"enum" => true, + .array => |info| isPlainData(info.child), + .optional => |info| isPlainData(info.child), + .@"struct" => |info| inline for (info.field_types) |Field| { + if (!isPlainData(Field)) break false; + } else true, + .@"union" => |info| info.tag_type != null and inline for (info.field_types) |Field| { + if (!isPlainData(Field)) break false; + } else true, + else => false, + }; + } }; pub const Source = @import("Source.zig"); @@ -109,8 +251,8 @@ pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, return self; } -/// Frees the bindings and the sources, then the registry entries `init` -/// took. +/// Frees the bindings, the sources and programs, then the registry entries +/// `init` took. pub fn deinit(self: *TensorStore) void { self.id_to_sources.deinit(self.allocator); self.arena.deinit(); @@ -146,8 +288,9 @@ pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mes } /// What model code holds: a key prefix over the store's checkpoint, to look -/// tensors up and bind them. A value that keeps its own copy of the prefix -/// (256 bytes at most, longer panics) and points to the store. +/// tensors up, bind them and set their memory. A value that keeps its own +/// copy of the prefix (256 bytes at most, longer panics) and points to the +/// store. pub const View = struct { store: *TensorStore, @@ -221,6 +364,42 @@ pub const View = struct { return self.createTensor(subkey, tags, .replicated, .{}); } + /// Changes where the loader delivers `tensor` (`Memory`), for a + /// decision made after the tensor was bound. `tensor` must be bound in + /// this store, and not computed by a program: a program's output lands + /// in default device memory. + pub fn setMemory(self: View, tensor: Tensor, kind: Memory) void { + const binding = self.store.id_to_sources.getPtr(tensor.id) orelse + stdx.debug.panic("Tensor {} {f} has no binding in this store", .{ tensor.id, tensor.shape() }); + switch (binding.*) { + .load => |*load| load.memory = kind, + .compute => |compute| stdx.debug.panic("setMemory: program {s} computes tensor {} {f}, whose output lands in default device memory", .{ compute.program.name(), tensor.id, tensor.shape() }), + } + } + + /// The bytes on one device of every tensor the store delivers + /// `.host_mapped`, each counted by its shard: what XLA counts against + /// device memory for those views, which a model gives back through + /// `Compiler.Options.extra_device_memory_bytes`. The whole store's, + /// whatever this view's prefix; ask once every binding and `setMemory` + /// is done. A tensor whose mesh partitions do not divide its dimensions + /// is `IncompatibleSharding`, logged. + pub fn hostMappedBytes(self: View) !u64 { + var bytes: u64 = 0; + var it = self.store.id_to_sources.iterator(); + while (it.next()) |entry| switch (entry.value_ptr.*) { + .load => |load| if (load.memory == .host_mapped) { + const shard = Sharding.shardedShape(load.source.shape) catch |err| { + load_log.err("tensor {} {f}, delivered host_mapped: its dimensions are not divisible by its mesh partitions", .{ entry.key_ptr.*, load.source.shape }); + return err; + }; + bytes += shard.byteSize(); + }, + .compute => {}, + }; + return bytes; + } + /// See `TensorStore.mesh`. pub fn mesh(self: View, name: @EnumLiteral()) *const Sharding.Mesh { return self.store.mesh(name); @@ -292,3 +471,88 @@ test "get finds exact keys, the registry's own entries, and count the keys under const tensor = layer.createReplicatedTensor("experts.0.down_proj.weight", null); try std.testing.expectEqual(entry, store.bindingOf(tensor.id).?.load.source.whole().?); } + +test "programs of one type and equal values over the same placed shapes share an executable" { + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "t", .shape = .init(.{4}, .u8), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + const Scale = struct { + factor: u8, + + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0]; + } + }; + const Shift = struct { + factor: u8, + + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0]; + } + }; + const shape: Shape = .init(.{4}, .u8); + const input = store.view().source("t", null).withPartitioning(.replicated, .{}); + const tensors = [_]Tensor{ + Source.bindProgram(&.{input}, shape, Scale{ .factor = 1 }), + Source.bindProgram(&.{input}, shape, Scale{ .factor = 1 }), + Source.bindProgram(&.{input}, shape, Scale{ .factor = 2 }), + Source.bindProgram(&.{input}, shape, Shift{ .factor = 1 }), + }; + var computes: [tensors.len]Binding.Compute = undefined; + for (&computes, tensors) |*compute, tensor| compute.* = store.bindingOf(tensor.id).?.compute; + try std.testing.expect(computes[0].sameExecutable(shape, computes[1], shape)); + try std.testing.expect(!computes[0].sameExecutable(shape, computes[2], shape)); + try std.testing.expect(!computes[0].sameExecutable(shape, computes[3], shape)); + try std.testing.expect(!computes[0].sameExecutable(shape, computes[1], shape.withDtype(.i8))); + try std.testing.expectEqual(1, computes[3].program.as(Shift).?.factor); + try std.testing.expect(computes[3].program.as(Scale) == null); +} + +test "setMemory changes where the loader delivers a bound tensor" { + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "table", .shape = .init(.{ 4, 2 }, .u8), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + + const view_ = store.view(); + const tensor = view_.createReplicatedTensor("table", null); + try std.testing.expectEqual(Memory.default, store.bindingOf(tensor.id).?.load.memory); + view_.setMemory(tensor, .host_mapped); + try std.testing.expectEqual(Memory.host_mapped, store.bindingOf(tensor.id).?.load.memory); + view_.setMemory(tensor, .default); + try std.testing.expectEqual(Memory.default, store.bindingOf(tensor.id).?.load.memory); +} + +test "hostMappedBytes counts one device's shard of each host_mapped tensor" { + const allocator = std.testing.allocator; + var devices: testing.Devices = undefined; + try devices.init(allocator, std.testing.io, 2); + defer devices.deinit(); + const tp = try devices.mesh("tp"); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "rows", .shape = .init(.{ 8, 2 }, .u8), .offset = 0 }); + try registry.registerTensor(.{ .file_uri = "weights", .name = "scale", .shape = .init(.{4}, .u8), .offset = 16 }); + try registry.registerTensor(.{ .file_uri = "weights", .name = "dense", .shape = .init(.{4}, .u8), .offset = 20 }); + try registry.registerTensor(.{ .file_uri = "weights", .name = "odd", .shape = .init(.{ 3, 2 }, .u8), .offset = 24 }); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + + const view_ = store.view(); + const rows = view_.createTensor("rows", .{ .row, .col }, .tp, .{ .row = .model }); + const scale = view_.createReplicatedTensor("scale", null); + _ = view_.createReplicatedTensor("dense", null); + try std.testing.expectEqual(0, try view_.hostMappedBytes()); + view_.setMemory(rows, .host_mapped); + view_.setMemory(scale, .host_mapped); + // Half the rows, and the whole replicated scale, from any prefix. + try std.testing.expectEqual(8 + 4, try view_.withPrefix("rows").hostMappedBytes()); + + // Three rows do not split over two devices. + const odd = view_.createTensor("odd", .{ .row, .col }, .tp, .{ .row = .model }); + view_.setMemory(odd, .host_mapped); + try std.testing.expectError(error.IncompatibleSharding, view_.hostMappedBytes()); +} From 0aee67518c73348d1fcbd2a07e0d3c367b5e8863 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:47:56 +0100 Subject: [PATCH 052/103] zml/io: add the buffered backend zml/io/buffered_loader.zig is the new loader's backend for the platforms that place a whole host tensor with Buffer.from: TPU, neuron and metal. It does what the old loader's BufferedMemoryWriter does, on the seam's LoadSpec. Nothing calls it yet; io.zig's test block reaches it. submit starts up to the read width of workers on a batch, which claim its specs from a shared cursor; on an io that starts no worker the batch loads inside submit. Each tensor allocates its host staging, zeroes the bytes no part covers and reads each contiguous part at its offset with Tensor.readRange. A strided part reads its source range once, past the tensor, and copyRuns puts each run where it lands. Buffer.from(wait) then places the tensor on its shape's mesh. Several batches can be in flight, each with its own workers. The first failure is kept and logged once, with the tensor's file, offset and sizes; workers then start no other tensor, every later submit returns it, and each output is left as it was or holds its whole buffer. abort fails the loader with Canceled. awaitBatch waits for the workers without a cancellation point, since they borrow the batch's specs. The tests submit specs directly on CPU devices. No machine available runs TPU, neuron or metal, so it has not run on one. Co-Authored-By: Claude Opus 5.5 --- zml/BUILD.bazel | 1 + zml/io.zig | 1 + zml/io/buffered_loader.zig | 342 +++++++++++++++++++++++++++++++++++++ 3 files changed, 344 insertions(+) create mode 100644 zml/io/buffered_loader.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 309c0213..aa33a4fc 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -90,6 +90,7 @@ zig_library( "io/Source.zig", "io/TensorStore.zig", "io/backend.zig", + "io/buffered_loader.zig", "io/log.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", diff --git a/zml/io.zig b/zml/io.zig index 20987cf1..34c6ae82 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -31,6 +31,7 @@ test { _ = @import("io/Source.zig"); _ = @import("io/TensorStore.zig"); _ = @import("io/backend.zig"); + _ = @import("io/buffered_loader.zig"); _ = @import("io/log.zig"); } diff --git a/zml/io/buffered_loader.zig b/zml/io/buffered_loader.zig new file mode 100644 index 00000000..844fe0b2 --- /dev/null +++ b/zml/io/buffered_loader.zig @@ -0,0 +1,342 @@ +//! Whole-tensor staging for platforms that use Buffer.from: TPU, neuron +//! and metal, until the direct path is checked on them. A composite +//! destination is assembled in the staging: its gaps zeroed, each +//! part read at its offset (a strided part reads its source range once and +//! copies each run where it lands), then the whole tensor is placed as any +//! other. Each batch runs up to the read width of workers over its specs; +//! each tensor allocates its host staging and frees it once placed. There +//! is no calibration and no pinned memory. The shared front end owns source +//! lookup, programs and admission. +const std = @import("std"); + +const stdx = @import("stdx"); + +const Buffer = @import("../buffer.zig").Buffer; +const Platform = @import("../platform.zig").Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const backend = @import("backend.zig"); +const testing = @import("../testing.zig"); +const LoadPart = backend.LoadPart; +const LoadSpec = backend.LoadSpec; +const log = @import("log.zig").load; + +pub const Loader = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + /// Workers per batch, so the threads grow with the batches in flight. + width: usize, + first_error: stdx.FirstError(anyerror) = .{}, + + pub fn create( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: backend.Options, + ) !*Loader { + const self = try allocator.create(Loader); + self.* = .{ + .allocator = allocator, + .io = io, + .platform = platform, + .width = opts.read_parallelism, + }; + return self; + } + + /// Starts up to `width` workers over the specs, each assembling one + /// destination at a time from its parts. The batch borrows `specs` and + /// everything they name until it is awaited (`backend.LoadSpec`). On an + /// io that starts no worker, the batch loads inside this call. Nothing + /// runs when this fails. + pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { + try self.checkOpen(); + const batch = try self.allocator.create(Batch); + batch.* = .{ .specs = specs, .progress = progress }; + // `concurrent`, never `async`: a busy Threaded io runs `async` + // inline, and an inline first worker would drain the whole cursor + // here. + var started: usize = 0; + for (0..@min(self.width, specs.len)) |_| { + batch.workers.concurrent(self.io, work, .{ self, batch }) catch break; + started += 1; + } + if (started == 0) self.work(batch); + return batch; + } + + /// Waits for the batch's workers, frees it and returns the sticky error. + /// Uncancelable: the workers borrow the batch and its specs until they + /// return. + pub fn awaitBatch(self: *Loader, batch: *Batch) !void { + const protection = self.io.swapCancelProtection(.blocked); + batch.workers.await(self.io) catch |err| switch (err) { + error.Canceled => unreachable, + }; + _ = self.io.swapCancelProtection(protection); + self.allocator.destroy(batch); + try self.checkOpen(); + } + + /// Fails the loader unless it already failed: workers start no other + /// tensor. + pub fn abort(self: *Loader) void { + self.recordError(error.Canceled, "loader aborted", .{}); + } + + /// Every batch was awaited, so no worker runs. + pub fn destroy(self: *Loader) void { + self.allocator.destroy(self); + } + + fn recordError(self: *Loader, err: anyerror, comptime fmt: []const u8, args: anytype) void { + if (self.first_error.set(err)) log.failure(err, fmt, args); + } + + fn checkOpen(self: *Loader) !void { + if (self.first_error.get()) |err| return err; + } + + /// One of the batch's workers: loads the next unclaimed spec until none + /// is left. + fn work(self: *Loader, batch: *Batch) void { + while (true) { + const index = batch.next.fetchAdd(1, .monotonic); + if (index >= batch.specs.len) return; + self.loadOne(batch.specs[index]); + // One item on the caller's node: a child node per tensor runs + // out of std.Progress's 127. + batch.progress.completeOne(); + } + } + + fn loadOne(self: *Loader, spec: LoadSpec) void { + if (self.first_error.get() != null) return; + std.debug.assert(spec.parts.len != 0); + const tensor_bytes = spec.shape.byteSize(); + const staging = self.allocator.alloc(u8, stagingBytes(spec)) catch |err| + return self.fail(err, "allocate staging for", spec, &spec.parts[0]); + defer self.allocator.free(staging); + const tensor = staging[0..tensor_bytes]; + // The staging's contents are undefined, so a byte no part writes + // would publish garbage rather than zero: the gaps are zeroed first. + var strided = false; + for (spec.parts) |part| strided = strided or part.lattice.dims.len != 0; + if (strided) { + @memset(tensor, 0); + } else { + // Only the gaps: the parts overwrite the rest. + var covered_end: usize = 0; + for (spec.parts) |part| { + const start: usize = @intCast(part.lattice.dst_offset); + @memset(tensor[covered_end..start], 0); + covered_end = start + @as(usize, @intCast(part.lattice.width)); + } + @memset(tensor[covered_end..], 0); + } + for (spec.parts) |*part| { + const file = std.Io.Dir.openFile(.cwd(), self.io, part.source.file_uri, .{ .mode = .read_only }) catch |err| + return self.fail(err, "open source for", spec, part); + defer file.close(self.io); + const lattice = part.lattice; + if (lattice.dims.len == 0) { + const start: usize = @intCast(lattice.dst_offset); + part.source.readRange(self.io, file, tensor[start..][0..@intCast(lattice.width)], lattice.src_offset) catch |err| + return self.fail(err, "read", spec, part); + continue; + } + // A strided part: its source range once, into the staging past + // the tensor, then each run where it lands. + const source = staging[tensor_bytes..][0..@intCast(lattice.spans().?.src)]; + part.source.readRange(self.io, file, source, lattice.src_offset) catch |err| + return self.fail(err, "read", spec, part); + lattice.copyRuns(0, lattice.runs(), @intFromPtr(tensor.ptr), @intFromPtr(source.ptr) -% @as(usize, @intCast(lattice.src_offset))); + } + // The front end asks no host kind of this backend. + std.debug.assert(spec.memory == .default); + // The shape's sharding places the bytes; a shape without a mesh is + // replicated, as `Buffer.from` places it. + spec.output.* = Buffer.from(self.io, self.platform, spec.shape, tensor, .{ .wait = true }) catch |err| + return self.fail(err, "transfer", spec, &spec.parts[0]); + } + + /// Fails the loader with `err`, which `step` of `spec` met at `part`. + fn fail(self: *Loader, err: anyerror, step: []const u8, spec: LoadSpec, part: *const LoadPart) void { + self.recordError(err, "{s} tensor {s}: file={s}, offset={d}, source_bytes={d}, parts={d}, destination_bytes={d}", .{ step, part.source.name, part.source.file_uri, part.source.offset, part.source.byteSize(), spec.parts.len, spec.shape.byteSize() }); + } +}; + +/// The staging one spec takes: its tensor, then room for its widest strided +/// part's source range, which is read once before its runs are copied out. +fn stagingBytes(spec: LoadSpec) usize { + var span: usize = 0; + for (spec.parts) |part| { + if (part.lattice.dims.len != 0) span = @max(span, @as(usize, @intCast(part.lattice.spans().?.src))); + } + return spec.shape.byteSize() + span; +} + +/// One buffered submission: its workers claim specs from `next`. +pub const Batch = struct { + specs: []const LoadSpec, + progress: std.Progress.Node, + next: std.atomic.Value(usize) = .init(0), + workers: std.Io.Group = .init, +}; + +test "buffered loader assembles a composite destination" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 2); + defer devices.deinit(); + const platform = devices.platform; + const mesh = try devices.mesh("tp"); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var a: [48]u8 = undefined; + testing.pattern(&a, 1); + var b: [32]u8 = undefined; + testing.pattern(&b, 2); + var full: [128]u8 = undefined; + testing.pattern(&full, 3); + try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &a }, + .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &b }, + .{ .name = "full", .dtype = "U16", .dims = &.{ 8, 8 }, .bytes = &full }, + }); + // Rows split over the two devices: b at row 1, a's last two rows (from + // byte 16 of a) at row 4, and a strided part, the left halves of full's + // rows 0 and 2, at rows 6 and 7, around a leading row, a gap and the + // right halves of zeros. + const shape = Shape.init(.{ .row = 8, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); + const columns = [_]backend.Lattice.Dim{.{ .count = 2, .src_stride = 32, .dst_stride = 16 }}; + const parts = [_]LoadPart{ + .{ .source = registry.tensors.getPtr("b").?, .lattice = .{ .dst_offset = 16, .width = 32 } }, + .{ .source = registry.tensors.getPtr("a").?, .lattice = .{ .src_offset = 16, .dst_offset = 64, .width = 32 } }, + .{ .source = registry.tensors.getPtr("full").?, .lattice = .{ .dst_offset = 96, .width = 8, .dims = &columns } }, + }; + try std.testing.expect(backend.checkParts(&parts, shape.byteSize())); + var expected: [128]u8 = @splat(0); + @memcpy(expected[16..48], &b); + @memcpy(expected[64..96], a[16..48]); + @memcpy(expected[96..104], full[0..8]); + @memcpy(expected[112..120], full[32..40]); + const whole = [_]LoadPart{.whole(registry.tensors.getPtr("full").?)}; + + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.destroy(); + var output: Buffer = undefined; + try loader.awaitBatch(try loader.submit(&.{.{ .shape = shape, .output = &output, .parts = &parts }}, .none)); + defer output.deinit(); + try testing.expectShards(allocator, io, &output, &expected, .default); + + // A source whose file does not exist fails the worker, and the error + // is sticky from then on. + var missing = registry.tensors.get("full").?; + missing.file_uri = "missing.safetensors"; + const missing_parts = [_]LoadPart{.whole(&missing)}; + var never_written: Buffer = undefined; + const batch = try loader.submit(&.{.{ .shape = shape, .output = &never_written, .parts = &missing_parts }}, .none); + try std.testing.expectError(error.FileNotFound, loader.awaitBatch(batch)); + // `expectError` would print the batch, and every type its specs name. + if (loader.submit(&.{.{ .shape = shape, .output = &never_written, .parts = &whole }}, .none)) |_| + return error.TestUnexpectedResult + else |err| + try std.testing.expectEqual(error.FileNotFound, err); +} + +test "an aborted loader leaves each output as it was or whole, and stays failed" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var bytes: [64]u8 = undefined; + testing.pattern(&bytes, 1); + try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "x", .dtype = "U8", .dims = &.{ 8, 8 }, .bytes = &bytes }, + }); + const shape = Shape.init(.{ .row = 8, .col = 8 }, .u8); + const parts = [_]LoadPart{.whole(registry.tensors.getPtr("x").?)}; + // One worker over many specs, so the abort lands among them. + var specs: [64]LoadSpec = undefined; + var outputs: [64]Buffer = undefined; + for (&specs, &outputs) |*spec, *output| { + output.* = .{ ._platform = devices.platform, ._shape = shape, ._shards = .empty }; + spec.* = .{ .shape = shape, .output = output, .parts = &parts }; + } + defer for (&outputs) |*output| output.deinit(); + + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 1 }); + defer loader.destroy(); + const batch = try loader.submit(&specs, .none); + loader.abort(); + try std.testing.expectError(error.Canceled, loader.awaitBatch(batch)); + for (&outputs) |*output| { + if (output._shards.len != 0) try testing.expectShards(allocator, io, output, &bytes, .default); + } + if (loader.submit(&specs, .none)) |_| + return error.TestUnexpectedResult + else |err| + try std.testing.expectEqual(error.Canceled, err); +} + +test "buffered loader runs overlapping batches over their cursors, with or without concurrency" { + const allocator = std.testing.allocator; + var devices: testing.Devices = undefined; + try devices.init(allocator, std.testing.io, 2); + defer devices.deinit(); + const mesh = try devices.mesh("tp"); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + // Tensors of 2 to 32 rows of 8 u16. + const count = 24; + var bytes: [count][512]u8 = undefined; + var names: [count][8]u8 = undefined; + var dims: [count][2]i64 = undefined; + var entries: [count]testing.TensorBytes = undefined; + for (&bytes, &names, &dims, &entries, 0..) |*data, *name, *dim, *entry, i| { + const rows = 2 * (i % 16 + 1); + testing.pattern(data[0 .. rows * 16], @intCast(i + 1)); + dim.* = .{ @intCast(rows), 8 }; + entry.* = .{ .name = try std.fmt.bufPrint(name, "t{d}", .{i}), .dtype = "U16", .dims = dim, .bytes = data[0 .. rows * 16] }; + } + try testing.writeSafetensors(allocator, std.testing.io, tmp.dir, "weights.safetensors", ®istry, &entries); + + var parts: [count][1]LoadPart = undefined; + var specs: [count]LoadSpec = undefined; + var outputs: [count]Buffer = undefined; + for (&parts, &specs, &outputs, entries) |*part, *spec, *output, entry| { + part[0] = .whole(registry.tensors.getPtr(entry.name).?); + const shape = Shape.init(.{ .row = entry.dims[0], .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); + spec.* = .{ .shape = shape, .output = output, .parts = part }; + } + + var no_concurrency: std.Io.Threaded = .init(allocator, .{ .async_limit = .nothing, .concurrent_limit = .nothing }); + defer no_concurrency.deinit(); + for ([_]std.Io{ std.testing.io, no_concurrency.io() }) |io| { + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 4 }); + defer loader.destroy(); + // Both batches are in flight before either is awaited, each with + // its own workers. + const first = try loader.submit(specs[0 .. count / 2], .none); + const second = try loader.submit(specs[count / 2 ..], .none); + try loader.awaitBatch(first); + try loader.awaitBatch(second); + defer for (&outputs) |*output| output.deinit(); + for (&outputs, entries) |*output, entry| try testing.expectShards(allocator, io, output, entry.bytes, .default); + } +} From ba53d7af7cd2a4868018d95ac151b7105e631a18 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:51:50 +0100 Subject: [PATCH 053/103] zml/io: bound buffered staging and reuse it best-fit The buffered backend allocated each tensor's host staging and freed it once placed, with nothing bounding the bytes but the worker count. Mapping, faulting and unmapping a whole tensor's pages for every tensor takes half of a serial buffered load of a 14.96 GiB checkpoint, and a fifth of a concurrent one. Staging keeps released buffers for reuse under one budget for the live and kept bytes alike: min(read width, 12) times the largest staging request submitted so far. A request takes the smallest kept buffer that fits. Otherwise it waits until its size fits beside the live bytes, frees kept buffers that are too small, smallest first, and allocates. Each submission can only raise the budget. A worker that waited for its staging loads nothing if the loader failed meanwhile. Reused staging holds another tensor's bytes, so the zeroing of gaps is now load-bearing; the composite test loads a bigger tensor first to check it. Co-Authored-By: Claude Opus 5.5 --- zml/io/buffered_loader.zig | 257 +++++++++++++++++++++++++++++++++++-- 1 file changed, 247 insertions(+), 10 deletions(-) diff --git a/zml/io/buffered_loader.zig b/zml/io/buffered_loader.zig index 844fe0b2..9a988e54 100644 --- a/zml/io/buffered_loader.zig +++ b/zml/io/buffered_loader.zig @@ -4,7 +4,7 @@ //! part read at its offset (a strided part reads its source range once and //! copies each run where it lands), then the whole tensor is placed as any //! other. Each batch runs up to the read width of workers over its specs; -//! each tensor allocates its host staging and frees it once placed. There +//! one byte budget bounds the host staging, live and kept for reuse. There //! is no calibration and no pinned memory. The shared front end owns source //! lookup, programs and admission. const std = @import("std"); @@ -21,12 +21,21 @@ const LoadPart = backend.LoadPart; const LoadSpec = backend.LoadSpec; const log = @import("log.zig").load; +/// The host staging budget in largest requests: this many times the +/// largest staging request submitted (`stagingBytes`), for the buffers in +/// use and those kept for reuse alike. Twelve rather than the read width: +/// at a high-latency width of 32 the read width would nearly triple the +/// host staging for nothing this path can use. +const staging_tensors: usize = 12; + pub const Loader = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, - /// Workers per batch, so the threads grow with the batches in flight. + /// Workers per batch, so the threads grow with the batches in flight: + /// the staging budget bounds their bytes, not their number. width: usize, + staging: Staging, first_error: stdx.FirstError(anyerror) = .{}, pub fn create( @@ -41,6 +50,7 @@ pub const Loader = struct { .io = io, .platform = platform, .width = opts.read_parallelism, + .staging = .{ .slots = @min(opts.read_parallelism, staging_tensors) }, }; return self; } @@ -52,8 +62,11 @@ pub const Loader = struct { /// runs when this fails. pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { try self.checkOpen(); + var largest: usize = 0; + for (specs) |spec| largest = @max(largest, stagingBytes(spec)); const batch = try self.allocator.create(Batch); batch.* = .{ .specs = specs, .progress = progress }; + self.staging.widen(self.io, largest); // `concurrent`, never `async`: a busy Threaded io runs `async` // inline, and an inline first worker would drain the whole cursor // here. @@ -80,13 +93,14 @@ pub const Loader = struct { } /// Fails the loader unless it already failed: workers start no other - /// tensor. + /// tensor, and one waiting for its staging returns once it gets it. pub fn abort(self: *Loader) void { self.recordError(error.Canceled, "loader aborted", .{}); } /// Every batch was awaited, so no worker runs. pub fn destroy(self: *Loader) void { + self.staging.deinit(self.allocator); self.allocator.destroy(self); } @@ -115,12 +129,15 @@ pub const Loader = struct { if (self.first_error.get() != null) return; std.debug.assert(spec.parts.len != 0); const tensor_bytes = spec.shape.byteSize(); - const staging = self.allocator.alloc(u8, stagingBytes(spec)) catch |err| + const staging = self.staging.acquire(self.allocator, self.io, stagingBytes(spec)) catch |err| return self.fail(err, "allocate staging for", spec, &spec.parts[0]); - defer self.allocator.free(staging); + defer self.staging.release(self.allocator, self.io, staging); + // The staging may have taken a while: a failure meanwhile leaves + // this tensor unstarted. + if (self.first_error.get() != null) return; const tensor = staging[0..tensor_bytes]; - // The staging's contents are undefined, so a byte no part writes - // would publish garbage rather than zero: the gaps are zeroed first. + // Staging is reused, so a byte no part writes would publish another + // tensor's bytes rather than zero: the gaps are zeroed first. var strided = false; for (spec.parts) |part| strided = strided or part.lattice.dims.len != 0; if (strided) { @@ -185,7 +202,172 @@ pub const Batch = struct { workers: std.Io.Group = .init, }; -test "buffered loader assembles a composite destination" { +/// Host staging for the buffered backend: one budget for the buffers the +/// running tensors hold and those kept for reuse, which are handed back on +/// completion instead of freed. Mapping, faulting and then unmapping a whole +/// tensor's worth of pages for every tensor takes half of a serial buffered +/// load of a 14.96 GiB checkpoint, and a fifth of a concurrent one. +/// +/// The budget is `slots` times the largest request submitted, so no +/// request is wider than it. A request takes the smallest kept buffer that +/// fits, which counts in full while it is held: small tensors run far wider +/// than the slot count on small buffers, but once only large ones are kept +/// they take those, and run about `slots` wide. +const Staging = struct { + mutex: std.Io.Mutex = .init, + room: std.Io.Condition = .init, + /// The budget in largest requests (`staging_tensors`). + slots: usize, + /// Zero until the first submission reports its largest request. + budget: usize = 0, + /// What the running tensors hold. + live_bytes: usize = 0, + /// Buffers no tensor holds, and their total length. + free: std.ArrayList([]u8) = .empty, + retained_bytes: usize = 0, + + /// Raises the budget to `slots` of `largest`. Submissions only ever widen + /// it: a later submission of smaller tensors cannot shrink what is + /// already running. + fn widen(self: *Staging, io: std.Io, largest: usize) void { + const wanted = largest *| self.slots; + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + if (wanted <= self.budget) return; + self.budget = wanted; + self.room.broadcast(io); + } + + /// A whole allocation of at least `size` bytes, with undefined contents: + /// the allocation itself, not a slice of one. The caller slices it for + /// its own use and returns this exact slice to `release`, whatever + /// happens. Waits until a retained buffer fits, or `size` more bytes + /// fit the budget beside the live ones. `size` is at most the budget: + /// `widen` ran for the request's submission. + fn acquire(self: *Staging, allocator: std.mem.Allocator, io: std.Io, size: usize) ![]u8 { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + std.debug.assert(size <= self.budget); + while (true) { + if (self.take(size)) |buffer| return buffer; + if (self.live_bytes + size <= self.budget) break; + self.room.waitUncancelable(io, &self.mutex); + } + // No retained buffer fits, so each is smaller than this request: + // free them, smallest first, until it fits beside the rest. + while (self.live_bytes + self.retained_bytes + size > self.budget) { + const dropped = self.free.swapRemove(self.smallest()); + self.retained_bytes -= dropped.len; + allocator.free(dropped); + } + const fresh = try allocator.alloc(u8, size); + self.live_bytes += fresh.len; + std.debug.assert(self.live_bytes + self.retained_bytes <= self.budget); + return fresh; + } + + /// `buffer` must be what `acquire` returned, not a slice of it. It is + /// kept: the budget already counts it. + fn release(self: *Staging, allocator: std.mem.Allocator, io: std.Io, buffer: []u8) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.live_bytes -= buffer.len; + self.room.broadcast(io); + self.free.append(allocator, buffer) catch return allocator.free(buffer); + self.retained_bytes += buffer.len; + } + + /// Every tensor is done, so every buffer is back. + fn deinit(self: *Staging, allocator: std.mem.Allocator) void { + std.debug.assert(self.live_bytes == 0); + for (self.free.items) |buffer| allocator.free(buffer); + self.free.deinit(allocator); + } + + /// The smallest retained buffer that fits, now live, so the big ones + /// stay free for the big tensors. Called with `mutex` held. + fn take(self: *Staging, size: usize) ?[]u8 { + var best: ?usize = null; + for (self.free.items, 0..) |buffer, i| { + if (buffer.len < size) continue; + if (best == null or buffer.len < self.free.items[best.?].len) best = i; + } + const buffer = self.free.swapRemove(best orelse return null); + self.retained_bytes -= buffer.len; + self.live_bytes += buffer.len; + return buffer; + } + + /// Called with `mutex` held and a buffer retained. + fn smallest(self: *const Staging) usize { + var index: usize = 0; + for (self.free.items, 0..) |buffer, i| { + if (buffer.len < self.free.items[index].len) index = i; + } + return index; + } +}; + +test "staging holds live and retained bytes within one budget" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var staging: Staging = .{ .slots = 3 }; + defer staging.deinit(allocator); + + // Three slots of the largest request is the budget, and a later, + // smaller submission cannot shrink it. + staging.widen(io, 16); + staging.widen(io, 4); + try std.testing.expectEqual(@as(usize, 48), staging.budget); + + // With nothing kept, small requests run far wider than the slot count, + // and their buffers are kept. + var small: [12][]u8 = undefined; + for (&small) |*buffer| buffer.* = try staging.acquire(allocator, io, 4); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + for (small) |buffer| staging.release(allocator, io, buffer); + try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); + + // A request no retained buffer fits frees as many as it needs room + // from. + const large = try staging.acquire(allocator, io, 16); + try std.testing.expectEqual(@as(usize, 32), staging.retained_bytes); + staging.release(allocator, io, large); + // The smallest buffer that fits comes back to a smaller request, so the + // 16 stays for one the 4s cannot serve. The first take moves the 16 to + // the front of the list, ahead of the second's 4. + var reused: [2][]u8 = undefined; + for (&reused) |*buffer| { + buffer.* = try staging.acquire(allocator, io, 3); + try std.testing.expectEqual(@as(usize, 4), buffer.len); + } + try std.testing.expectEqual(@as(usize, 40), staging.retained_bytes); + for (reused) |buffer| staging.release(allocator, io, buffer); + try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); + + // A full budget holds the next request until a buffer comes back, then + // hands it that buffer. + var held: [3][]u8 = undefined; + for (&held) |*buffer| buffer.* = try staging.acquire(allocator, io, 16); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes + staging.retained_bytes); + var waiting = try io.concurrent(Staging.acquire, .{ &staging, allocator, io, 16 }); + try io.sleep(.fromMilliseconds(5), .awake); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + staging.release(allocator, io, held[0]); + held[0] = try waiting.await(io); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + for (held) |buffer| staging.release(allocator, io, buffer); + + // Once only large buffers are kept, small requests take them, each + // counted in full: three fill the budget. + var on_large: [3][]u8 = undefined; + for (&on_large) |*buffer| buffer.* = try staging.acquire(allocator, io, 4); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + for (on_large) |buffer| staging.release(allocator, io, buffer); + try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); +} + +test "buffered loader assembles a composite destination in reused staging" { const allocator = std.testing.allocator; const io = std.testing.io; var devices: testing.Devices = undefined; @@ -204,10 +386,13 @@ test "buffered loader assembles a composite destination" { testing.pattern(&b, 2); var full: [128]u8 = undefined; testing.pattern(&full, 3); + var big: [256]u8 = undefined; + testing.pattern(&big, 4); try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &a }, .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &b }, .{ .name = "full", .dtype = "U16", .dims = &.{ 8, 8 }, .bytes = &full }, + .{ .name = "big", .dtype = "U16", .dims = &.{ 16, 8 }, .bytes = &big }, }); // Rows split over the two devices: b at row 1, a's last two rows (from // byte 16 of a) at row 4, and a strided part, the left halves of full's @@ -227,13 +412,26 @@ test "buffered loader assembles a composite destination" { @memcpy(expected[96..104], full[0..8]); @memcpy(expected[112..120], full[32..40]); const whole = [_]LoadPart{.whole(registry.tensors.getPtr("full").?)}; + const big_shape = Shape.init(.{ .row = 16, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); + const big_whole = [_]LoadPart{.whole(registry.tensors.getPtr("big").?)}; const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2 }); defer loader.destroy(); + // A bigger tensor first, so the composite, its strided source range + // included, reuses its staging buffer and only the zeroing keeps its + // bytes out of the gaps. + var first: Buffer = undefined; + try loader.awaitBatch(try loader.submit(&.{.{ .shape = big_shape, .output = &first, .parts = &big_whole }}, .none)); + defer first.deinit(); + try testing.expectShards(allocator, io, &first, &big, .default); var output: Buffer = undefined; try loader.awaitBatch(try loader.submit(&.{.{ .shape = shape, .output = &output, .parts = &parts }}, .none)); defer output.deinit(); try testing.expectShards(allocator, io, &output, &expected, .default); + // One buffer was ever allocated, within the budget of two slots of the + // largest request. + try std.testing.expectEqual(@as(usize, 512), loader.staging.budget); + try std.testing.expectEqual(@as(usize, 256), loader.staging.retained_bytes); // A source whose file does not exist fails the worker, and the error // is sticky from then on. @@ -250,6 +448,41 @@ test "buffered loader assembles a composite destination" { try std.testing.expectEqual(error.FileNotFound, err); } +test "a worker waiting for its staging loads nothing once the loader failed" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var bytes: [64]u8 = undefined; + testing.pattern(&bytes, 1); + try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "x", .dtype = "U8", .dims = &.{ 8, 8 }, .bytes = &bytes }, + }); + const shape = Shape.init(.{ .row = 8, .col = 8 }, .u8); + const parts = [_]LoadPart{.whole(registry.tensors.getPtr("x").?)}; + + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 1 }); + defer loader.destroy(); + // The test holds the whole budget, one slot of the tensor, so the + // worker waits for its staging until the loader failed. + loader.staging.widen(io, bytes.len); + const held = try loader.staging.acquire(allocator, io, bytes.len); + var output: Buffer = .{ ._platform = devices.platform, ._shape = shape, ._shards = .empty }; + defer output.deinit(); + const batch = try loader.submit(&.{.{ .shape = shape, .output = &output, .parts = &parts }}, .none); + try io.sleep(.fromMilliseconds(5), .awake); + loader.abort(); + loader.staging.release(allocator, io, held); + try std.testing.expectError(error.Canceled, loader.awaitBatch(batch)); + try std.testing.expectEqual(@as(usize, 0), output._shards.len); +} + test "an aborted loader leaves each output as it was or whole, and stays failed" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -302,7 +535,8 @@ test "buffered loader runs overlapping batches over their cursors, with or witho defer tmp.cleanup(); var registry: safetensors.TensorRegistry = .init(allocator); defer registry.deinit(); - // Tensors of 2 to 32 rows of 8 u16. + // Tensors of 2 to 32 rows of 8 u16, so the staging budget holds a + // varying number of them. const count = 24; var bytes: [count][512]u8 = undefined; var names: [count][8]u8 = undefined; @@ -331,12 +565,15 @@ test "buffered loader runs overlapping batches over their cursors, with or witho const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 4 }); defer loader.destroy(); // Both batches are in flight before either is awaited, each with - // its own workers. + // its own workers: the staging budget bounds their bytes + // (`Staging.acquire` asserts it). const first = try loader.submit(specs[0 .. count / 2], .none); const second = try loader.submit(specs[count / 2 ..], .none); try loader.awaitBatch(first); try loader.awaitBatch(second); defer for (&outputs) |*output| output.deinit(); for (&outputs, entries) |*output, entry| try testing.expectShards(allocator, io, output, entry.bytes, .default); + try std.testing.expect(loader.staging.retained_bytes <= loader.staging.budget); + try std.testing.expectEqual(@as(usize, 0), loader.staging.live_bytes); } } From 0d614777e949daebeb0b196ac75778ee62da7d8d Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:57:11 +0100 Subject: [PATCH 054/103] zml/io: split positional reads on high-latency sources The buffered backend read each part with one positional read, so its read concurrency was only its worker count, and one large tensor read as a single stream. A remote source answers a request after a round trip however small it is, so a tensor split over parallel range requests loads faster there. ReadPermits bounds the reads in flight across every task of the backend to the read width. A tensor's read takes one permit, then, when the VFS read hints mark the source high-latency, grabs the permits that are free, without waiting, and starts a helper on each. The owner and its helpers claim spans from a shared byte cursor (ChunkedRead), each the tensor split as many ways as it has readers but never below the hints' read_chunk_size, so a lone reader still takes the whole tensor in one read. A helper that the io does not start gives its permit back. Permits are released before the tensor's transfer, so a tensor placing on the devices lends its read budget to one still reading. On a local source a tensor stays one read: splitting there only spends tasks. The tests check the claim sizes, run overlapping batches with 32-byte chunks with and without concurrency, and load a tensor from a MockServer HTTP origin in exactly four range GETs over four permits. Co-Authored-By: Claude Opus 5.5 --- zml/io/buffered_loader.zig | 279 +++++++++++++++++++++++++++++++++++-- 1 file changed, 269 insertions(+), 10 deletions(-) diff --git a/zml/io/buffered_loader.zig b/zml/io/buffered_loader.zig index 9a988e54..c3d2e5e2 100644 --- a/zml/io/buffered_loader.zig +++ b/zml/io/buffered_loader.zig @@ -4,12 +4,13 @@ //! part read at its offset (a strided part reads its source range once and //! copies each run where it lands), then the whole tensor is placed as any //! other. Each batch runs up to the read width of workers over its specs; -//! one byte budget bounds the host staging, live and kept for reuse. There -//! is no calibration and no pinned memory. The shared front end owns source -//! lookup, programs and admission. +//! read permits bound the source reads and one byte budget the host +//! staging, live and kept for reuse. There is no calibration and no pinned +//! memory. The shared front end owns source lookup, programs and admission. const std = @import("std"); const stdx = @import("stdx"); +const VFS = @import("vfs"); const Buffer = @import("../buffer.zig").Buffer; const Platform = @import("../platform.zig").Platform; @@ -25,7 +26,8 @@ const log = @import("log.zig").load; /// largest staging request submitted (`stagingBytes`), for the buffers in /// use and those kept for reuse alike. Twelve rather than the read width: /// at a high-latency width of 32 the read width would nearly triple the -/// host staging for nothing this path can use. +/// host staging for nothing this path can use (on such a source a tensor +/// splits its read across the free permits). const staging_tensors: usize = 12; pub const Loader = struct { @@ -33,8 +35,19 @@ pub const Loader = struct { io: std.Io, platform: *const Platform, /// Workers per batch, so the threads grow with the batches in flight: - /// the staging budget bounds their bytes, not their number. + /// the read permits bound their reads and the staging budget their + /// bytes, not their number. width: usize, + /// The smallest claim of a tensor's read split across helpers + /// (`ChunkedRead.claim`). A tensor is one positional read unless spare + /// permits split it on a high-latency source. + read_chunk_size: usize, + /// The widest fan-out one tensor may take. One on a source whose reads + /// are bandwidth-bound rather than round-trip bound: splitting there only + /// spends tasks, and measurably so -- a replicated local load, whose long + /// transfers keep permits free, loses time to helpers it has no use for. + tensor_workers: usize, + permits: ReadPermits, staging: Staging, first_error: stdx.FirstError(anyerror) = .{}, @@ -51,6 +64,11 @@ pub const Loader = struct { .platform = platform, .width = opts.read_parallelism, .staging = .{ .slots = @min(opts.read_parallelism, staging_tensors) }, + .read_chunk_size = opts.read_hints.read_chunk_size, + // A high-latency source may use spare read permits to split a + // tensor already staging: more concurrency without more host memory. + .tensor_workers = if (opts.read_hints.high_latency) opts.read_parallelism else 1, + .permits = .{ .free = opts.read_parallelism }, }; return self; } @@ -159,14 +177,14 @@ pub const Loader = struct { const lattice = part.lattice; if (lattice.dims.len == 0) { const start: usize = @intCast(lattice.dst_offset); - part.source.readRange(self.io, file, tensor[start..][0..@intCast(lattice.width)], lattice.src_offset) catch |err| + self.readInto(part.source, file, lattice.src_offset, tensor[start..][0..@intCast(lattice.width)]) catch |err| return self.fail(err, "read", spec, part); continue; } // A strided part: its source range once, into the staging past // the tensor, then each run where it lands. const source = staging[tensor_bytes..][0..@intCast(lattice.spans().?.src)]; - part.source.readRange(self.io, file, source, lattice.src_offset) catch |err| + self.readInto(part.source, file, lattice.src_offset, source) catch |err| return self.fail(err, "read", spec, part); lattice.copyRuns(0, lattice.runs(), @intFromPtr(tensor.ptr), @intFromPtr(source.ptr) -% @as(usize, @intCast(lattice.src_offset))); } @@ -182,6 +200,73 @@ pub const Loader = struct { fn fail(self: *Loader, err: anyerror, step: []const u8, spec: LoadSpec, part: *const LoadPart) void { self.recordError(err, "{s} tensor {s}: file={s}, offset={d}, source_bytes={d}, parts={d}, destination_bytes={d}", .{ step, part.source.name, part.source.file_uri, part.source.offset, part.source.byteSize(), spec.parts.len, spec.shape.byteSize() }); } + + /// Fills `destination` from `source`'s bytes at `source_offset`, read + /// from `file`, splitting it across helper tasks when the source permits + /// are free. The permits are released before the caller's transfer, so a + /// tensor staging to the device lends its read budget to one still + /// reading. + fn readInto( + self: *Loader, + source: *const safetensors.Tensor, + file: std.Io.File, + source_offset: u64, + destination: []u8, + ) safetensors.Tensor.RangeError!void { + var chunked: ChunkedRead = .{ + .io = self.io, + .source = source, + .file = file, + .source_offset = source_offset, + .destination = destination, + .chunk_size = self.read_chunk_size, + }; + // Splitting below the read chunk buys nothing, so that is the + // most workers this tensor can use. + var useful_workers = @min( + self.tensor_workers, + destination.len / self.read_chunk_size + + @intFromBool(destination.len % self.read_chunk_size != 0), + ); + + self.permits.acquire(self.io); + var held: usize = 1; + var group: std.Io.Group = .init; + while (true) { + // Helpers only ever take permits that are already free, so a busy + // source sees one read per tensor. The check repeats between + // claims because the tensors that finish first hand their budget + // to the one still reading -- which is the long tensor that + // needed the help. + // Nothing unclaimed means a helper would only start and stop. + if (held < useful_workers and chunked.next.load(.monotonic) < destination.len) { + const extra = self.permits.tryAcquire(self.io, useful_workers - held); + if (extra != 0) { + // Claims size themselves by the width, so it is raised + // before a helper can claim. `concurrent`, since a busy + // io runs `async` inline, on this task. + chunked.workers.store(held + extra, .monotonic); + var started: usize = 0; + while (started < extra) : (started += 1) { + group.concurrent(self.io, ChunkedRead.run, .{&chunked}) catch break; + } + held += started; + if (started < extra) { + // The io starts no more: the rest of the permits go + // back, and this tensor stops asking. + self.permits.release(self.io, extra - started); + chunked.workers.store(held, .monotonic); + useful_workers = held; + } + } + } + if (!chunked.readOne()) break; + } + group.await(self.io) catch {}; + self.permits.release(self.io, held); + + if (chunked.failure.get()) |err| return err; + } }; /// The staging one spec takes: its tensor, then room for its widest strided @@ -202,6 +287,100 @@ pub const Batch = struct { workers: std.Io.Group = .init, }; +/// Source reads in flight across every task of the buffered backend, at most +/// `read_parallelism`. On a high-latency source, a tensor wider than the +/// hints' `read_chunk_size` may split its read across the permits that are +/// free, for parallel streams. +const ReadPermits = struct { + mutex: std.Io.Mutex = .init, + available: std.Io.Condition = .init, + free: usize, + + fn acquire(self: *ReadPermits, io: std.Io) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (self.free == 0) self.available.waitUncancelable(io, &self.mutex); + self.free -= 1; + } + + /// Up to `count` permits, without waiting: how many were free is how much + /// concurrency this tensor gets, so a loaded source is left alone. + fn tryAcquire(self: *ReadPermits, io: std.Io, count: usize) usize { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + const taken = @min(count, self.free); + self.free -= taken; + return taken; + } + + fn release(self: *ReadPermits, io: std.Io, count: usize) void { + if (count == 0) return; + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.free += count; + for (0..count) |_| self.available.signal(io); + } +}; + +/// One tensor's read, claimed from a shared byte cursor by the owning task +/// and its helpers. Positional reads leave the file as it was, so they can +/// run concurrently on one file. +/// +/// A claim is the tensor split `workers` ways, and never smaller than the +/// hints' `read_chunk_size` -- that field is a minimum request size, and +/// on a high-latency source a request costs a round trip whether or not it +/// runs beside another. So one worker takes the whole tensor in a single +/// read, and only the concurrency the source can actually carry turns into +/// extra requests. +const ChunkedRead = struct { + io: std.Io, + source: *const safetensors.Tensor, + file: std.Io.File, + /// Where `destination` starts in the tensor's bytes. + source_offset: u64 = 0, + destination: []u8, + chunk_size: usize, + /// Reads this tensor may have in flight: the owner raises it before it + /// starts helpers on newly taken permits, and lowers it to the helpers + /// that started. + workers: std.atomic.Value(usize) = .init(1), + /// Bytes already claimed. + next: std.atomic.Value(usize) = .init(0), + failure: stdx.FirstError(safetensors.Tensor.RangeError) = .{}, + + fn run(self: *ChunkedRead) void { + while (self.readOne()) {} + } + + /// Reads the next unclaimed span. False once the tensor is fully claimed, + /// or once a read failed. + fn readOne(self: *ChunkedRead) bool { + if (self.failure.get() != null) return false; + const span = self.claim() orelse return false; + self.source.readRange(self.io, self.file, span.bytes, self.source_offset + span.offset) catch |err| { + _ = self.failure.set(err); + return false; + }; + return true; + } + + fn claim(self: *ChunkedRead) ?struct { offset: usize, bytes: []u8 } { + var start = self.next.load(.monotonic); + while (start < self.destination.len) { + const workers = self.workers.load(.monotonic); + const share = self.destination.len / workers + + @intFromBool(self.destination.len % workers != 0); + const len = @min(@max(self.chunk_size, share), self.destination.len - start); + if (self.next.cmpxchgWeak(start, start + len, .monotonic, .monotonic)) |actual| { + start = actual; + continue; + } + return .{ .offset = start, .bytes = self.destination[start..][0..len] }; + } + return null; + } +}; + /// Host staging for the buffered backend: one budget for the buffers the /// running tensors hold and those kept for reuse, which are handed back on /// completion instead of freed. Mapping, faulting and then unmapping a whole @@ -308,6 +487,39 @@ const Staging = struct { } }; +test "chunked read splits a tensor only as wide as it has workers" { + var destination: [1000]u8 = undefined; + var work: ChunkedRead = .{ + .io = undefined, + .source = undefined, + .file = undefined, + .destination = &destination, + .chunk_size = 100, + }; + + // One worker reads the whole tensor, exactly as an unsplit read would. + const whole = work.claim().?; + try std.testing.expectEqual(@as(usize, 0), whole.offset); + try std.testing.expectEqual(@as(usize, 1000), whole.bytes.len); + try std.testing.expect(work.claim() == null); + + // Four workers take a quarter each. + work.next = .init(0); + work.workers = .init(4); + for (0..4) |i| { + const span = work.claim().?; + try std.testing.expectEqual(i * 250, span.offset); + try std.testing.expectEqual(@as(usize, 250), span.bytes.len); + } + try std.testing.expect(work.claim() == null); + + // Never below the read chunk, however many workers there are. + work.next = .init(0); + work.workers = .init(50); + const floored = work.claim().?; + try std.testing.expectEqual(@as(usize, 100), floored.bytes.len); +} + test "staging holds live and retained bytes within one budget" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -562,11 +774,16 @@ test "buffered loader runs overlapping batches over their cursors, with or witho var no_concurrency: std.Io.Threaded = .init(allocator, .{ .async_limit = .nothing, .concurrent_limit = .nothing }); defer no_concurrency.deinit(); for ([_]std.Io{ std.testing.io, no_concurrency.io() }) |io| { - const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 4 }); + // A high-latency source with 32-byte chunks: helpers split each + // tensor whenever a permit is free. + const loader = try Loader.create(allocator, io, devices.platform, .{ + .read_parallelism = 4, + .read_hints = .{ .read_chunk_size = 32, .high_latency = true }, + }); defer loader.destroy(); // Both batches are in flight before either is awaited, each with - // its own workers: the staging budget bounds their bytes - // (`Staging.acquire` asserts it). + // its own workers: the permits bound their reads and the staging + // budget their bytes (`Staging.acquire` asserts it). const first = try loader.submit(specs[0 .. count / 2], .none); const second = try loader.submit(specs[count / 2 ..], .none); try loader.awaitBatch(first); @@ -575,5 +792,47 @@ test "buffered loader runs overlapping batches over their cursors, with or witho for (&outputs, entries) |*output, entry| try testing.expectShards(allocator, io, output, entry.bytes, .default); try std.testing.expect(loader.staging.retained_bytes <= loader.staging.budget); try std.testing.expectEqual(@as(usize, 0), loader.staging.live_bytes); + try std.testing.expectEqual(@as(usize, 4), loader.permits.free); } } + +test "a tensor from a high-latency HTTP source splits its read over the free permits" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + + var object: [4096]u8 = undefined; + testing.pattern(&object, 1); + var server = try VFS.MockServer.init(io, &object, .{}); + defer server.deinit(io); + try server.start(io); + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var http: VFS.HTTP = try .init(allocator, io, &client, .{ .protocol = .http }); + defer http.deinit(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + try vfs.registerBackend("http", http.backend()); + + var url_buffer: [64]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + const source: safetensors.Tensor = .{ .file_uri = url, .name = "x", .shape = .init(.{object.len}, .u8), .offset = 0 }; + const parts = [_]LoadPart{.whole(&source)}; + + // Four free permits and a quarter of the tensor as the read chunk: the + // worker and three helpers each read a quarter, one range GET each. + const loader = try Loader.create(allocator, vfs.io(), devices.platform, .{ + .read_parallelism = 4, + .read_hints = .{ .read_chunk_size = object.len / 4, .high_latency = true }, + }); + defer loader.destroy(); + var output: Buffer = undefined; + try loader.awaitBatch(try loader.submit(&.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, .none)); + defer output.deinit(); + try server.stop(io); + try testing.expectShards(allocator, io, &output, &object, .default); + try std.testing.expectEqual(@as(usize, 4), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 4), loader.permits.free); +} From a1eda19435c2b881bad99b17291ebb360117594d Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:08:43 +0100 Subject: [PATCH 055/103] zml/mem: pair tensors and buffers by structure pairBufferized walks a model and the buffer tree bufferize made from it together, in meta.visit order, and returns one Pair per tensor: the tensor, the buffer at its place and its field path from the root, such as .layers[1].ffn.weight. The pairs and paths live in the caller's arena. The two trees must have the structure bufferize gives them. At the first place they differ (another union arm, an optional set on one side only, slices of other lengths) the walk stops with error.Mismatch, and the report writer holds the path there and what differs. Comparing tensor counts alone would let two trees of equal counts and different shapes pair the wrong buffers. The new loader's front end is its first caller. Co-Authored-By: Claude Opus 5.5 --- zml/mem.zig | 189 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/zml/mem.zig b/zml/mem.zig index 888e8064..dab7a87d 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -380,6 +380,98 @@ pub inline fn bufferize(allocator: std.mem.Allocator, model: anytype) !Bufferize return bufferized; } +/// A tensor of a model, the buffer at its place in the model's buffers, and +/// its field path from the root, as `.layers[1].ffn.weight`. +pub const Pair = struct { tensor: *const Tensor, buffer: *Buffer, path: []const u8 }; + +/// Pairs each tensor of the model `model` points to with the buffer at its +/// place in `bufferized`, in the order `meta.visit` finds the tensors. The +/// pairs and their paths live in `arena`. The two trees must have the +/// structure `bufferize` gives: the same union arms, the same optionals set, +/// slices of the same lengths. `report`, an empty fixed writer, builds the +/// paths and cuts them short when full. At the first place the trees differ +/// the walk stops and returns `error.Mismatch`, and `report` ends with the +/// path there and what differs. +pub fn pairBufferized( + arena: std.mem.Allocator, + model: anytype, + bufferized: *Bufferized(@TypeOf(model.*)), + report: *std.Io.Writer, +) error{ Mismatch, OutOfMemory }![]Pair { + var pairs: std.ArrayList(Pair) = try .initCapacity(arena, meta.count(Tensor, model)); + try pairBufferizedInner(arena, model, bufferized, report, &pairs); + return pairs.items; +} + +/// `report.end` is restored after each part that pairs, so at a tensor or a +/// mismatch the report holds the path down to it. +fn pairBufferizedInner( + arena: std.mem.Allocator, + model: anytype, + bufferized: anytype, + report: *std.Io.Writer, + pairs: *std.ArrayList(Pair), +) error{ Mismatch, OutOfMemory }!void { + const Model = @typeInfo(@TypeOf(model)).pointer.child; + if (comptime !meta.Contains(Model, Tensor)) return; + if (Model == Tensor) return pairs.appendAssumeCapacity(.{ + .tensor = model, + // `bufferize` allocates every slice mutable, a `[]const` one included. + .buffer = @constCast(bufferized), + .path = try arena.dupe(u8, report.buffered()), + }); + + const depth = report.end; + switch (@typeInfo(Model)) { + .@"struct" => |info| inline for (info.field_names) |name| { + if (comptime bufferizedField(Model, name)) |buffer_field| { + report.print(".{s}", .{name}) catch {}; + try pairBufferizedInner(arena, &@field(model, name), &@field(bufferized, buffer_field), report, pairs); + report.end = depth; + } + }, + .@"union" => { + const tag = std.meta.activeTag(model.*); + if (tag != std.meta.activeTag(bufferized.*)) { + report.print(": union arm .{t} in the model, .{t} in the buffers", .{ tag, std.meta.activeTag(bufferized.*) }) catch {}; + return error.Mismatch; + } + switch (model.*) { + inline else => |*payload, arm| { + report.print(".{s}", .{@tagName(arm)}) catch {}; + try pairBufferizedInner(arena, payload, &@field(bufferized, @tagName(arm)), report, pairs); + report.end = depth; + }, + } + }, + .optional => { + if ((model.* == null) != (bufferized.* == null)) { + report.print(": {s} in the model, {s} in the buffers", .{ if (model.* == null) "null" else "set", if (bufferized.* == null) "null" else "set" }) catch {}; + return error.Mismatch; + } + if (model.*) |*payload| try pairBufferizedInner(arena, payload, &bufferized.*.?, report, pairs); + }, + .pointer => |info| { + if (info.size != .slice) @compileError("mem.pairBufferized: " ++ @typeName(Model) ++ " is not a slice"); + if (model.len != bufferized.len) { + report.print(": a slice of {d} in the model, of {d} in the buffers", .{ model.len, bufferized.len }) catch {}; + return error.Mismatch; + } + for (model.*, bufferized.*, 0..) |*element, *buffer, i| { + report.print("[{d}]", .{i}) catch {}; + try pairBufferizedInner(arena, element, buffer, report, pairs); + report.end = depth; + } + }, + .array => for (model, bufferized, 0..) |*element, *buffer, i| { + report.print("[{d}]", .{i}) catch {}; + try pairBufferizedInner(arena, element, buffer, report, pairs); + report.end = depth; + }, + else => @compileError("mem.pairBufferized: cannot walk " ++ @typeName(Model)), + } +} + /// Deinitializes every accelerator buffer of the tree `value` points to and /// frees the recursive slice storage allocated by `bufferize`. pub fn deinitBufferized(allocator: std.mem.Allocator, value: anytype) void { @@ -484,3 +576,100 @@ test "deinitBufferized frees slices under optionals and unions" { var bufferized = try bufferize(std.testing.allocator, &model); deinitBufferized(std.testing.allocator, &bufferized); } + +test "pairBufferized pairs tensors and buffers in visit order" { + const Layer = struct { weights: []Tensor, scale: ?Tensor }; + const Ffn = union(enum) { dense: Layer, experts: []Layer, none: void }; + const Model = struct { ffns: [3]Ffn, head: Tensor, layers: u32 }; + var weights: [4]Tensor = undefined; + var experts = [_]Layer{ .{ .weights = weights[1..2], .scale = null }, .{ .weights = weights[2..4], .scale = weights[3] } }; + var model: Model = .{ + .ffns = .{ .{ .dense = .{ .weights = weights[0..1], .scale = weights[0] } }, .{ .experts = &experts }, .none }, + .head = undefined, + .layers = 2, + }; + var bufferized = try bufferize(std.testing.allocator, &model); + defer deinitBufferized(std.testing.allocator, &bufferized); + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + var report_buffer: [64]u8 = undefined; + var report: std.Io.Writer = .fixed(&report_buffer); + const pairs = try pairBufferized(arena.allocator(), &model, &bufferized, &report); + try std.testing.expectEqual(0, report.end); + + const Visited = struct { + tensors: [8]*const Tensor = undefined, + buffers: [8]*Buffer = undefined, + tensor_count: usize = 0, + buffer_count: usize = 0, + }; + var visited: Visited = .{}; + meta.forEachVisit(&model, *const Tensor, struct { + fn call(_: usize, tensor: *const Tensor, out: *Visited) void { + out.tensors[out.tensor_count] = tensor; + out.tensor_count += 1; + } + }.call, .{&visited}); + meta.forEachVisit(&bufferized, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, out: *Visited) void { + out.buffers[out.buffer_count] = buffer; + out.buffer_count += 1; + } + }.call, .{&visited}); + const paths = [_][]const u8{ + ".ffns[0].dense.weights[0]", + ".ffns[0].dense.scale", + ".ffns[1].experts[0].weights[0]", + ".ffns[1].experts[1].weights[0]", + ".ffns[1].experts[1].weights[1]", + ".ffns[1].experts[1].scale", + ".head", + }; + try std.testing.expectEqual(paths.len, pairs.len); + try std.testing.expectEqual(pairs.len, visited.tensor_count); + try std.testing.expectEqual(pairs.len, visited.buffer_count); + // Not `expectEqualSlices`: printing a `*Buffer` reaches code a CPU test build lacks. + for (pairs, paths, 0..) |pair, path, i| { + try std.testing.expect(pair.tensor == visited.tensors[i]); + try std.testing.expect(pair.buffer == visited.buffers[i]); + try std.testing.expectEqualStrings(path, pair.path); + } +} + +test "pairBufferized stops at the first place the trees differ and names it" { + const Layer = struct { weights: []Tensor, scale: ?Tensor }; + const Ffn = union(enum) { dense: Layer, none: void }; + const Model = struct { ffns: []Ffn, head: Tensor }; + var weights: [3]Tensor = undefined; + var built = [_]Ffn{ .{ .dense = .{ .weights = weights[0..2], .scale = null } }, .none }; + const built_model: Model = .{ .ffns = &built, .head = undefined }; + var bufferized = try bufferize(std.testing.allocator, &built_model); + defer deinitBufferized(std.testing.allocator, &bufferized); + + const Case = struct { ffns: [2]Ffn, report: []const u8 }; + const cases = [_]Case{ + .{ .ffns = .{ .none, .none }, .report = ".ffns[0]: union arm .none in the model, .dense in the buffers" }, + .{ + .ffns = .{ .{ .dense = .{ .weights = weights[0..2], .scale = weights[2] } }, .none }, + .report = ".ffns[0].dense.scale: set in the model, null in the buffers", + }, + .{ + .ffns = .{ .{ .dense = .{ .weights = weights[0..1], .scale = null } }, .none }, + .report = ".ffns[0].dense.weights: a slice of 1 in the model, of 2 in the buffers", + }, + }; + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + for (cases) |case| { + var ffns = case.ffns; + const model: Model = .{ .ffns = &ffns, .head = undefined }; + var report_buffer: [128]u8 = undefined; + var report: std.Io.Writer = .fixed(&report_buffer); + // Not `expectError`: printing a `*Buffer` reaches code a CPU test build lacks. + if (pairBufferized(arena.allocator(), &model, &bufferized, &report)) |_| { + return error.TestExpectedMismatch; + } else |err| try std.testing.expectEqual(error.Mismatch, err); + try std.testing.expectEqualStrings(case.report, report.buffered()); + } +} From e891faad572676a9f669275b3b6ca9714c529cf0 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:09:13 +0100 Subject: [PATCH 056/103] zml/io: add the Backend and Submission unions with the buffered arm The new loader's front end reaches its backend through backend.Backend: init, submit, await, abort and destroy, dispatched with inline else, and Submission, the batch a submit published. The union has one arm, the buffered backend, so every load of the new loader is buffered. supportsMemory says which delivery kinds the loader takes on a platform: default device memory only, since the buffered backend places whole tensors with Buffer.from. LoadSpec's doc now names it. Co-Authored-By: Claude Opus 5.5 --- zml/io/backend.zig | 74 +++++++++++++++++++++++++++++++++++--- zml/io/buffered_loader.zig | 3 +- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/zml/io/backend.zig b/zml/io/backend.zig index b58e21f6..3aec6782 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -1,15 +1,18 @@ -//! The seam between the loader's front end and its backends: the -//! `LoadSpec` contract they trust, its parts' byte runs (`Lattice`), and the -//! source request size (`requestSize`). +//! The seam between the front end (loader.zig) and its backends: the +//! `LoadSpec` contract they trust, its parts' byte runs (`Lattice`), the +//! source request size (`requestSize`), the backend a load runs on +//! (`Backend`) and the memory kinds it takes (`supportsMemory`). const std = @import("std"); const VFS = @import("vfs"); const Buffer = @import("../buffer.zig").Buffer; +const Platform = @import("../platform.zig").Platform; const safetensors = @import("../safetensors.zig"); const Shape = @import("../shape.zig").Shape; const Memory = @import("TensorStore.zig").Memory; const constants = @import("../constants.zig"); +const buffered_loader = @import("buffered_loader.zig"); /// The largest source request, and so the largest DMA block: a request is /// a whole number of blocks, at least one. @@ -71,8 +74,9 @@ pub const LoadSpec = struct { /// sharding places the bytes on the devices; a shape without a mesh is /// replicated over every device (`Platform.placedShape`). shape: Shape, - /// Where the bytes go. A host kind comes only with contiguous parts - /// (`Lattice.dims` empty). + /// Where the bytes go, from the source's `TensorStore` binding. The + /// front end only submits kinds `supportsMemory` accepts, and a host + /// kind only with contiguous parts (`Lattice.dims` empty). memory: Memory = .default, /// Written whole once the destination's buffers exist, and not touched /// before: a failure leaves it as it was or holding those buffers. @@ -257,6 +261,66 @@ pub fn checkParts(parts: []const LoadPart, destination_bytes: u64) bool { return covered <= destination_bytes; } +/// Whether the loader can deliver into `kind` on `platform`: device memory, +/// on every platform. The host kinds need a backend that writes host +/// memory in place. +pub fn supportsMemory(platform: *const Platform, kind: Memory) bool { + _ = platform; + return kind == .default; +} + +/// The transfer implementation a load runs on: the buffered backend, which +/// places each whole tensor with `Buffer.from`. +pub const Backend = union(enum) { + buffered: *buffered_loader.Loader, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Options, + ) !Backend { + return .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, opts) }; + } + + /// Publishes `specs`, and completes one item of `progress` per spec. + /// Fails with what the backend detected in the files, host memory and + /// PJRT, logged there (a plain allocation failure is `OutOfMemory`, + /// unlogged). + pub fn submit(self: Backend, specs: []const LoadSpec, progress: std.Progress.Node) !Submission { + return switch (self) { + inline else => |loader, tag| @unionInit(Submission, @tagName(tag), try loader.submit(specs, progress)), + }; + } + + /// Waits for and retires `submission`, which this backend published. + /// Its batch is dangling afterwards. + pub fn await(self: Backend, submission: Submission) !void { + return switch (self) { + inline else => |loader, tag| loader.awaitBatch(@field(submission, @tagName(tag))), + }; + } + + /// Fails the backend unless it already failed: queued work is dropped, + /// work in flight drains, and every later await returns the first error. + pub fn abort(self: Backend) void { + switch (self) { + inline else => |loader| loader.abort(), + } + } + + pub fn destroy(self: Backend) void { + switch (self) { + inline else => |loader| loader.destroy(), + } + } +}; + +/// A published batch of the backend that published it. +pub const Submission = union(enum) { + buffered: *buffered_loader.Batch, +}; + /// A random lattice whose runs land in order (`Lattice.dstOrdered`), its /// source strides anything, zero (a broadcast) included. fn randomLattice(random: std.Random, dims: []Lattice.Dim) Lattice { diff --git a/zml/io/buffered_loader.zig b/zml/io/buffered_loader.zig index c3d2e5e2..96cfac3e 100644 --- a/zml/io/buffered_loader.zig +++ b/zml/io/buffered_loader.zig @@ -188,7 +188,8 @@ pub const Loader = struct { return self.fail(err, "read", spec, part); lattice.copyRuns(0, lattice.runs(), @intFromPtr(tensor.ptr), @intFromPtr(source.ptr) -% @as(usize, @intCast(lattice.src_offset))); } - // The front end asks no host kind of this backend. + // The front end asks no host kind of this backend + // (`backend.supportsMemory`). std.debug.assert(spec.memory == .default); // The shape's sharding places the bytes; a shape without a mesh is // replicated, as `Buffer.from` places it. From 7d19e813fa6cfbf6578c3bc640649320016e10e4 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:13:18 +0100 Subject: [PATCH 057/103] zml/io: add Loader init, start, wait, cancel and deinit with one front-end check zml/io/loader.zig is the new loader's front end, on the buffered backend. zml does not export it: io.zig's old Loader still serves every caller, and only io.zig's test block reaches the new file. Loader.init checks the read width (1 to 128) and creates the backend. start takes a whole model, its buffer tree and the store that bound it. It pairs tensors with buffers by structure (mem.pairBufferized), so buffers bufferized from another tree are InvalidBinding with the field path where they differ, and it empties every buffer before anything is written. Each bound tensor then becomes one LoadSpec, checked once: its source is lowered to byte parts, an empty source or a run inside a byte is refused, the placement must pack sub-byte elements into whole bytes, divide over its mesh and name only the platform's devices, the memory kind must be one the platform takes (device memory), and the parts go through backend.checkParts. A source of strided runs (a transpose, a broadcast, an interleave or interior padding) and a tensor a program computes are refused: this loader assembles contiguous parts and runs no program. Every check failure is InvalidBinding, logged with the tensor's field path and its source's key. The specs are published as one submission; several starts, from one store or several, may come before one wait. wait retires the submissions in publish order and logs the bytes, time and rate once; it is idempotent. Every error of start or wait fails the loader: the backend is aborted, the pending submissions are retired, and every later call returns the first error. cancel does the same with Canceled. After any failure each buffer is empty or whole, and mem.deinitBufferized frees the tree. deinit panics on loads still started, whose store and buffers may be gone. Co-Authored-By: Claude Opus 5.5 --- zml/BUILD.bazel | 1 + zml/io.zig | 1 + zml/io/TensorStore.zig | 11 +- zml/io/loader.zig | 749 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 758 insertions(+), 4 deletions(-) create mode 100644 zml/io/loader.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index aa33a4fc..b889c8fc 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -91,6 +91,7 @@ zig_library( "io/TensorStore.zig", "io/backend.zig", "io/buffered_loader.zig", + "io/loader.zig", "io/log.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", diff --git a/zml/io.zig b/zml/io.zig index 34c6ae82..cca17b52 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -32,6 +32,7 @@ test { _ = @import("io/TensorStore.zig"); _ = @import("io/backend.zig"); _ = @import("io/buffered_loader.zig"); + _ = @import("io/loader.zig"); _ = @import("io/log.zig"); } diff --git a/zml/io/TensorStore.zig b/zml/io/TensorStore.zig index bc40c775..4d6043c5 100644 --- a/zml/io/TensorStore.zig +++ b/zml/io/TensorStore.zig @@ -6,9 +6,11 @@ //! //! The store owns those entries, the bindings, and every source and program //! value (its arena). Views and sources point to it: once `view` was -//! called it must not move, and it must outlive them. It borrows the meshes -//! of `Options.meshes`, which belong to the platform. Binding cannot fail: -//! a missing key panics with the key. +//! called it must not move, and it must outlive them and every load +//! started from it (`Loader.start` borrows it until `wait` or `cancel`). +//! It borrows the meshes of `Options.meshes`, which belong to the platform. +//! Binding cannot fail: a missing key panics with the key, and what the +//! loader cannot fill is its error (`Loader.start`'s `InvalidBinding`). const std = @import("std"); const stdx = @import("stdx"); @@ -252,7 +254,8 @@ pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, } /// Frees the bindings, the sources and programs, then the registry entries -/// `init` took. +/// `init` took. No load started from the store may still run: `wait` or +/// `cancel` it first. pub fn deinit(self: *TensorStore) void { self.id_to_sources.deinit(self.allocator); self.arena.deinit(); diff --git a/zml/io/loader.zig b/zml/io/loader.zig new file mode 100644 index 00000000..1740f6d6 --- /dev/null +++ b/zml/io/loader.zig @@ -0,0 +1,749 @@ +//! Checkpoint loading. `Loader.start` takes a whole model and publishes +//! every tensor it fills as one submission; `wait` retires it. Backends +//! only read sources into buffers; lookup and the checks stay in this +//! shared front end. The contract is in the pub doc comments below. +//! +//! From bindings to specs. `start` checks every tensor once, before +//! anything is published (`prepareSpec`, `checkPlacement`). It lowers each +//! bound source to bytes (`Source.Placed.lower`, the one rule), one part +//! per piece: a part of the destination's logical bytes (the row-major +//! bytes of its shape, before the sharding splits them), a run of `width` +//! bytes repeated over strided dimensions. A whole checkpoint tensor is one +//! part. Each `LoadSpec` is one destination, which the backends trust as +//! `backend.LoadSpec` states. Only `Source` makes bindings and its pieces +//! are disjoint by construction: nothing here checks that (Source's tests +//! do). +const std = @import("std"); + +const Buffer = @import("../buffer.zig").Buffer; +const mem = @import("../mem.zig"); +const Bufferized = mem.Bufferized; +const meta = @import("../meta.zig"); +const platform_mod = @import("../platform.zig"); +const Memory = TensorStore.Memory; +const Platform = platform_mod.Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; +const testing = @import("../testing.zig"); +const backend = @import("backend.zig"); +const Backend = backend.Backend; +const LoadSpec = backend.LoadSpec; +const load_log = @import("log.zig").load; +const TensorStore = @import("TensorStore.zig"); + +const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; + +/// Loads models from their checkpoint sources. +/// +/// What it borrows. The platform outlives the loader and every buffer it +/// loads. `io` outlives the loader: the backend's tasks run on it and open +/// the source files through it. `start` borrows the store, the buffers and +/// the progress node until `wait` or `cancel` returns, and reads the model +/// during the call only. The loaded buffers are the caller's and outlive +/// the loader. +/// +/// Start and wait serially on the owning task; source reads and transfers +/// run concurrently inside the backend. Submissions are retired in publish +/// order. +/// +/// What fails. Every error of `start` and `wait` fails the loader: +/// everything started is canceled (`cancel`), and every later call returns +/// the first error. Each is logged once, at err, where it is detected, but +/// for a plain allocation failure (`OutOfMemory` from an allocator) and +/// `Canceled`, the caller's own `cancel`, which are returned unlogged. A +/// model whose bindings are wrong is `InvalidBinding`, logged with the +/// tensor's field path, the key its source reads (none for a source that +/// reads nothing or a run inside a byte) and the rule it breaks; buffers +/// shaped unlike the model (`mem.pairBufferized`) too, logged with the +/// field path where they differ. The other errors a caller can act on: a +/// missing checkpoint file is `FileNotFound`; a denied one `AccessDenied` +/// locally and `PermissionDenied` remotely; a file that ends inside a +/// tensor `UnexpectedEndOfFile`, because it changed after its header was +/// read; host memory the backend cannot allocate `OutOfMemory`; device +/// memory that runs out PJRT's `ResourceExhausted`; `cancel` `Canceled`. +/// The rest (PJRT, a remote that failed) passes through as its source +/// names it. +pub const Loader = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + backend: Backend, + /// Submissions published and not yet retired, oldest first. + pending: std.Deque(PendingSubmission) = .empty, + /// The first error; every later call returns it. + failure: ?anyerror = null, + stats: Stats = .{}, + + pub const Options = struct { + /// Concurrent source reads, 1 to `max_read_parallelism`, fixed for + /// the load. Local checkpoints read fastest at 12 to 16 on every + /// host measured. + read_parallelism: usize = 16, + }; + + /// Concurrent source reads at most (`Options.read_parallelism`). + pub const max_read_parallelism: usize = 128; + + /// Checks `opts` and creates the backend. A read width out of range is + /// `InvalidOptions`, logged. + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Options, + ) !Loader { + const backend_opts = try backendOptions(opts); + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .backend = try .init(allocator, io, platform, backend_opts), + }; + } + + /// Gives up on everything started: aborts the backend (queued reads are + /// dropped, reads in flight finish) and retires the pending + /// submissions. Each buffer of the started loads is then empty or holds + /// its shards, whose contents are unspecified (unwritten, errored, or + /// partly written); read none, and free the tree as after a load + /// (`mem.deinitBufferized`). The loader is failed afterwards. For an + /// error between `start` and `wait`; nothing to do when nothing was + /// started. + pub fn cancel(self: *Loader) void { + if (self.loadsStarted()) self.fail(error.Canceled); + } + + /// Destroys the backend: its workers and its host staging. The buffers + /// it loaded stay the caller's. Panics on loads neither waited for nor + /// canceled. + pub fn deinit(self: *Loader) void { + if (self.loadsStarted()) std.debug.panic("Loader.deinit with loads started, whose store and buffers may be freed already: add `errdefer loader.cancel();` after `start`", .{}); + self.pending.deinit(self.allocator); + self.backend.destroy(); + self.* = undefined; + } + + /// Whether loads are started and neither waited for nor canceled. + fn loadsStarted(self: *const Loader) bool { + return self.pending.len != 0; + } + + /// Starts loading every tensor of `model` into its buffer in `buffers`, + /// as `store` binds it, and returns; `wait` returns once it is done. + /// Each tensor is placed by its shape's sharding (replicated without a + /// mesh), into the memory its binding names, assembled from the pieces + /// of its source (`Source.Placed.bind`) with every byte no piece covers + /// set to zero. All of these are published here, as one submission. + /// + /// Every tensor is checked once, before anything is published: a tensor + /// `store` did not bind, a tensor a program computes + /// (`Source.bindProgram`, which this loader does not run), a source + /// that reads no bytes (an empty checkpoint tensor among them), a source + /// that this loader cannot assemble (a run that starts or ends inside a + /// byte, or strided runs: a transpose, a broadcast, an interleave or + /// interior padding), a sub-byte last dimension that does not pack into + /// whole bytes, a placement its mesh does not divide or on devices the + /// platform lacks, and a memory kind the platform cannot take are + /// `error.InvalidBinding`. + /// + /// `buffers` is `mem.bufferize` of `model`, whose buffers are empty. A + /// tree of another structure (other union arms, optionals or slice + /// lengths: `mem.pairBufferized`) is `InvalidBinding`, logged with the + /// field path, before any buffer is written. `start` empties them + /// again, and each stays empty until its load writes it whole. So + /// after any error each buffer is empty or holds shards for the caller + /// to free: `mem.deinitBufferized` frees the tree either way. `buffers` + /// and `store` are borrowed until `wait` or `cancel` returns; do not end + /// `progress` before then. `start` adds one item to `progress` for each + /// tensor, a composite one once; the backend completes them. + /// Several loads may be started before one `wait`, from one store or + /// several. + pub fn start( + self: *Loader, + model: anytype, + buffers: *Bufferized(@TypeOf(model.*)), + store: *const TensorStore, + progress: std.Progress.Node, + ) !void { + if (self.failure) |err| return err; + self.startModel(model, buffers, store, progress) catch |err| { + self.fail(err); + return err; + }; + } + + fn startModel( + self: *Loader, + model: anytype, + buffers: *Bufferized(@TypeOf(model.*)), + store: *const TensorStore, + progress: std.Progress.Node, + ) !void { + self.stats.begin(self.io); + var prepared = try prepareModel(self.allocator, self.platform, store, model, buffers); + errdefer prepared.arena.deinit(); + progress.increaseEstimatedTotalItems(prepared.specs.len); + if (prepared.specs.len == 0) { + prepared.arena.deinit(); + return; + } + try self.submit(prepared.specs, prepared.arena, progress); + } + + /// Retires every submission in publish order. Returns once every + /// started load is in its buffers, and logs one info line: the bytes + /// loaded, the time since the first `start` and their rate. Idempotent: + /// with nothing started it returns at once and logs nothing. + pub fn wait(self: *Loader) !void { + if (self.failure) |err| return err; + self.drain() catch |err| { + self.fail(err); + return err; + }; + } + + fn drain(self: *Loader) !void { + while (self.pending.len != 0) try self.retireOldest(); + self.stats.end(self.io); + } + + /// Records `err` unless the loader failed already, and gives up on + /// everything started (`cancel`). The one writer of `failure`. + fn fail(self: *Loader, err: anyerror) void { + if (self.failure == null) self.failure = err; + // Nothing still pending can succeed. + if (self.pending.len != 0) self.backend.abort(); + while (self.pending.len != 0) self.retireOldest() catch {}; + } + + /// One submission over `specs`, which live in `arena`. Takes `arena` + /// once the submission is published; on failure the caller still owns + /// it. + fn submit( + self: *Loader, + specs: []const LoadSpec, + arena: std.heap.ArenaAllocator, + progress: std.Progress.Node, + ) !void { + // One spec per destination, however many parts fill it: its bytes + // count once. + var logical_bytes: usize = 0; + for (specs) |spec| logical_bytes += spec.shape.byteSize(); + try self.pending.ensureUnusedCapacity(self.allocator, 1); + const submission = try self.backend.submit(specs, progress); + self.pending.pushBackAssumeCapacity(.{ + .submission = submission, + .arena = arena, + .logical_bytes = logical_bytes, + }); + self.stats.submissions += 1; + } + + /// Retires the oldest pending submission: waits for its reads and + /// transfers, and counts its bytes once they landed. + fn retireOldest(self: *Loader) !void { + const oldest = self.pending.popFront().?; + defer oldest.arena.deinit(); + try self.backend.await(oldest.submission); + self.stats.bytes_loaded += oldest.logical_bytes; + } +}; + +/// The loader's counters: read only by logs and tests; nothing here +/// decides. +const Stats = struct { + /// When the first `start` since the last `wait` ran: `wait` logs the + /// load's time from there. + started_at: ?std.Io.Timestamp = null, + /// Logical bytes of the submissions retired since that `start`: each + /// destination's shape once, so a composite tensor counts its padding + /// and not its sources one by one. + bytes_loaded: usize = 0, + /// Submissions published since `init`. + submissions: usize = 0, + + /// Times and counts a load from the first `start` since the last + /// `wait`; later starts add to it. + fn begin(self: *Stats, io: std.Io) void { + if (self.started_at != null) return; + self.started_at = .now(io, .awake); + self.bytes_loaded = 0; + } + + /// Logs `wait`'s info line, unless nothing was started, and ends the + /// load's timing. + fn end(self: *Stats, io: std.Io) void { + const started_at = self.started_at orelse return; + const took = started_at.untilNow(io, .awake); + const seconds = @as(f64, @floatFromInt(@max(took.nanoseconds, 1))) / std.time.ns_per_s; + const rate: u64 = @intFromFloat(@as(f64, @floatFromInt(self.bytes_loaded)) / seconds); + load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ self.bytes_loaded, took, rate }); + self.started_at = null; + } +}; + +/// One published submission: what to wait for and what its accounting +/// owes. +const PendingSubmission = struct { + submission: backend.Submission, + /// The specs, which the backend reads until the submission is awaited + /// (`backend.LoadSpec`). + arena: std.heap.ArenaAllocator, + logical_bytes: usize, +}; + +/// A model as `start` takes it: one spec per tensor, in `arena` with the +/// specs' parts and the tensors' field paths. +const PreparedModel = struct { + arena: std.heap.ArenaAllocator, + specs: []const LoadSpec, +}; + +/// Pairs each tensor of `model` with its buffer and binding: each tensor +/// takes one spec (`prepareSpec`). A tensor a program computes is +/// `InvalidBinding`: this loader runs no programs. +fn prepareModel( + allocator: std.mem.Allocator, + platform: *const Platform, + store: *const TensorStore, + model: anytype, + buffers: *Bufferized(@TypeOf(model.*)), +) PrepareError!PreparedModel { + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + // Tensors and buffers pair by structure, before any buffer is written: a + // buffer tree shaped otherwise is refused whole. + var report_buffer: [256]u8 = undefined; + var report: std.Io.Writer = .fixed(&report_buffer); + const pairs = mem.pairBufferized(arena.allocator(), model, buffers, &report) catch |err| switch (err) { + error.Mismatch => { + load_log.err("buffers shaped unlike the model at model{s}; bufferize them from this model", .{report.buffered()}); + return error.InvalidBinding; + }, + error.OutOfMemory => return error.OutOfMemory, + }; + // Empty until the backend writes it whole, so the caller can free the + // tree whatever fails. + for (pairs) |pair| pair.buffer.* = .{ ._platform = platform, ._shape = pair.tensor.shape(), ._shards = .empty }; + + var specs: std.ArrayList(LoadSpec) = try .initCapacity(arena.allocator(), pairs.len); + for (pairs) |pair| { + const tensor = pair.tensor; + const target: Target = .{ .path = pair.path, .shape = tensor.shape() }; + const binding = store.bindingOf(tensor.id) orelse { + load_log.err("{f} has no checkpoint binding", .{target}); + return error.InvalidBinding; + }; + switch (binding) { + .load => |load| specs.appendAssumeCapacity(try prepareSpec(arena.allocator(), platform, target, load.source, load.memory, pair.buffer)), + .compute => |compute| { + load_log.err("{f} is computed by program {s}, which this loader does not run", .{ target, compute.program.name() }); + return error.InvalidBinding; + }, + } + } + return .{ .arena = arena, .specs = specs.items }; +} + +/// What a spec fills, as the logs name it: the model's tensor at `path` +/// (`mem.Pair.path`) of `shape`. +const Target = struct { + path: []const u8, + shape: Shape, + + pub fn format(self: Target, writer: *std.Io.Writer) !void { + try writer.print("model{s} {f}", .{ self.path, self.shape }); + } +}; + +/// The spec filling `output`, the buffer of `target`, from `source` into +/// `memory`, its parts in `arena`: the source lowered to bytes +/// (`Source.Placed.lower`), in destination order. The one check of what +/// the backends trust (`backend.LoadSpec`); a rule it breaks is +/// `InvalidBinding`, logged with the target, and with the key its source +/// reads once the source is lowered. +fn prepareSpec( + arena: std.mem.Allocator, + platform: *const Platform, + target: Target, + source: TensorStore.Source.Placed, + memory: Memory, + output: *Buffer, +) PrepareError!LoadSpec { + const shape = target.shape; + const parts = source.lower(arena) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.SubBytePosition => { + load_log.err("{f}: its source {f} reads a run that starts or ends inside a byte, which the loader cannot place", .{ target, source.shape }); + return error.InvalidBinding; + }, + }; + if (parts.len == 0) { + load_log.err("{f} reads no checkpoint bytes", .{target}); + return error.InvalidBinding; + } + const key = parts[0].source.name; + try checkPlacement(platform, target, key); + if (!backend.supportsMemory(platform, memory)) { + load_log.err("{f} from {s} is bound to {t} memory, which {t} cannot take", .{ target, key, memory, platform.target }); + return error.InvalidBinding; + } + for (parts) |part| { + if (part.lattice.dims.len != 0) { + load_log.err("{f} from {s}: a part of {d} strided runs of {d} bytes (a transpose, a broadcast, an interleave or interior padding), which this loader does not assemble", .{ target, part.source.name, part.lattice.runs(), part.lattice.width }); + return error.InvalidBinding; + } + } + if (!backend.checkParts(parts, shape.byteSize())) { + load_log.err("{f} from {s}: its parts leave their sources or its {d} bytes, pass the largest file offset, or come out of order", .{ target, key, shape.byteSize() }); + return error.InvalidBinding; + } + return .{ .shape = shape, .memory = memory, .output = output, .parts = parts }; +} + +/// Checks that `target`, from `from` (its source's key), places on +/// `platform`'s devices, in any memory space: a sub-byte last dimension +/// packs into whole bytes (`Shape.packedShape`), its mesh partitions divide +/// its dimensions, and every device of its mesh is one of the platform's. +fn checkPlacement(platform: *const Platform, target: Target, from: []const u8) error{InvalidBinding}!void { + const shape = target.shape; + const bits = shape.dtype().bitSizeOf(); + if (bits < 8 and (shape.rank() == 0 or @rem(shape.dim(-1), 8 / bits) != 0)) { + load_log.err("{f} from {s}: its last dimension does not pack into whole bytes", .{ target, from }); + return error.InvalidBinding; + } + const placed = platform.placedShape(shape).packedShape(); + const placement = Sharding.Placement.init(&placed) catch { + load_log.err("{f} from {s}: its dimensions are not divisible by its mesh partitions", .{ target, from }); + return error.InvalidBinding; + }; + for (placement.mesh().?.devicesInCanonicalOrder()) |device| { + if (device.id >= platform.devices.len) { + load_log.err("{f} from {s} is placed on device {d}, but {t} has {d}", .{ target, from, device.id, platform.target, platform.devices.len }); + return error.InvalidBinding; + } + } +} + +/// `opts` as the backends take them, with the local read hints. Every +/// option is checked here, once. +fn backendOptions(opts: Loader.Options) error{InvalidOptions}!backend.Options { + if (opts.read_parallelism == 0 or opts.read_parallelism > Loader.max_read_parallelism) { + load_log.err("invalid loader options: read_parallelism={d}, expected 1..{d}: InvalidOptions", .{ opts.read_parallelism, Loader.max_read_parallelism }); + return error.InvalidOptions; + } + return .{ .read_parallelism = opts.read_parallelism }; +} + +/// A four-byte tensor `value`, a four-byte `second`, a `missing` entry in a +/// file that does not exist, an `empty` tensor and a CPU platform. Pinned +/// after `init`. +const LoaderTestFixture = struct { + const contents = [_]u8{ 1, 2, 3, 4 }; + const second_contents = [_]u8{ 5, 6, 7, 8 }; + + tmp: std.testing.TmpDir, + store: TensorStore, + platform: *const Platform, + value: Tensor, + second: Tensor, + missing: Tensor, + empty: Tensor, + + fn init(self: *LoaderTestFixture, allocator: std.mem.Allocator, io: std.Io) !void { + self.tmp = std.testing.tmpDir(.{}); + errdefer self.tmp.cleanup(); + const file = try self.tmp.dir.createFile(io, "weights.bin", .{ .read = true }); + try file.writePositionalAll(io, &(contents ++ second_contents), 0); + var path_buffer: [1024]u8 = undefined; + const path_len = try file.realPath(io, &path_buffer); + file.close(io); + const path = path_buffer[0..path_len]; + var missing_buffer: [1024]u8 = undefined; + const missing_path = try std.fmt.bufPrint(&missing_buffer, "{s}.missing", .{path}); + + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ + .file_uri = path, + .name = "value", + .shape = .init(.{contents.len}, .u8), + .offset = 0, + }); + try registry.registerTensor(.{ + .file_uri = path, + .name = "second", + .shape = .init(.{second_contents.len}, .u8), + .offset = contents.len, + }); + try registry.registerTensor(.{ + .file_uri = missing_path, + .name = "missing", + .shape = .init(.{contents.len}, .u8), + .offset = 0, + }); + try registry.registerTensor(.{ + .file_uri = path, + .name = "empty", + .shape = .init(.{0}, .u8), + .offset = 0, + }); + self.platform = testing.env(); + self.store = .init(allocator, ®istry, .{ .meshes = &.{self.platform.replicated_mesh} }); + self.value = self.store.view().createReplicatedTensor("value", null); + self.second = self.store.view().createReplicatedTensor("second", null); + self.missing = self.store.view().createReplicatedTensor("missing", null); + self.empty = self.store.view().createReplicatedTensor("empty", null); + } + + fn deinit(self: *LoaderTestFixture) void { + self.store.deinit(); + self.tmp.cleanup(); + } + + /// A tensor of `like`'s shape that `program` computes from the + /// checkpoint tensor `name`. + fn computed(self: *LoaderTestFixture, name: []const u8, like: Tensor, program: anytype) Tensor { + const input = self.store.view().source(name, null).withPartitioning(.replicated, .{}); + return TensorStore.Source.bindProgram(&.{input}, like.shape(), program); + } + + /// The program returning its one input. + const Identity = struct { + pub fn forward(_: Identity, _: Shape, inputs: []const Tensor) Tensor { + return inputs[0]; + } + }; + + /// Whether `loader` failed with `expected` and holds nothing started. + fn expectFailed(loader: *const Loader, expected: anyerror) !void { + try std.testing.expectEqual(expected, loader.failure.?); + try std.testing.expectEqual(0, loader.pending.len); + } +}; + +test "start publishes the model at once and wait retires it" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Model = struct { value: Tensor, second: Tensor }; + const model: Model = .{ .value = fixture.value, .second = fixture.second }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + try std.testing.expectEqual(1, loader.pending.len); + try loader.wait(); + try std.testing.expectEqual(0, loader.pending.len); + try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &buffers.second, &LoaderTestFixture.second_contents, .default); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 2, loader.stats.bytes_loaded); + + // Idempotent: a second wait neither retires nor recounts. + try loader.wait(); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 2, loader.stats.bytes_loaded); + + // The next load counts and times from its own first start. + try std.testing.expectEqual(null, loader.stats.started_at); + var again: Buffer = undefined; + try loader.start(&fixture.second, &again, &fixture.store, .none); + try loader.wait(); + defer again.deinit(); + try std.testing.expectEqual(LoaderTestFixture.second_contents.len, loader.stats.bytes_loaded); + try std.testing.expectEqual(null, loader.stats.started_at); +} + +test "cancel gives up on everything started" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Model = struct { value: Tensor, second: Tensor }; + const model: Model = .{ .value = fixture.value, .second = fixture.second }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + loader.cancel(); + try LoaderTestFixture.expectFailed(&loader, error.Canceled); + // The loader is failed: what it waits for and what it is given. + try std.testing.expectError(error.Canceled, loader.wait()); + try std.testing.expectError(error.Canceled, loader.start(&model, &buffers, &fixture.store, .none)); + try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); +} + +test "a read failure fails the loader for every later call" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Model = struct { broken: Tensor, value: Tensor }; + const model: Model = .{ .broken = fixture.missing, .value = fixture.value }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + try std.testing.expectError(error.FileNotFound, loader.start(&model, &buffers, &fixture.store, .none)); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); +} + +test "after a failed load every buffer is empty or whole, and deinitBufferized frees the tree" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + // A tree with a slice and optionals, some of it in the file that loads, + // and a tensor in the file that does not exist. + const Layer = struct { weight: Tensor, bias: ?Tensor }; + const Model = struct { layers: []Layer, broken: Tensor }; + var layers = [_]Layer{ + .{ .weight = fixture.value, .bias = fixture.second }, + .{ .weight = fixture.second, .bias = null }, + }; + const model: Model = .{ .layers = &layers, .broken = fixture.missing }; + var buffers = try mem.bufferize(allocator, &model); + // Frees whatever the failed load wrote, under the testing allocator. + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + // Each buffer holds no shard or one per device. + const devices = fixture.platform.placedShape(fixture.value.shape())._sharding.mesh.?.devicesInCanonicalOrder().len; + var torn: usize = 0; + meta.forEachVisit(&buffers, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, shards: usize, count: *usize) void { + if (buffer._shards.len != 0 and buffer._shards.len != shards) count.* += 1; + } + }.call, .{ devices, &torn }); + try std.testing.expectEqual(0, torn); +} + +test "init checks every option and returns InvalidOptions" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + for ([_]usize{ 0, Loader.max_read_parallelism + 1 }) |width| { + if (Loader.init(allocator, io, platform, .{ .read_parallelism = width })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidOptions, err); + } +} + +test "a rejected binding fails the loader" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const platform = fixture.platform; + // A mesh of devices the platform does not have: the same tensors placed + // on it, in device and host memory. + var foreign_physical = platform.replicated_mesh.physical.*; + const foreign_devices = try allocator.dupe(Sharding.Device, foreign_physical.devices_in_canonical_order); + defer allocator.free(foreign_devices); + for (foreign_devices) |*device| device.id += 100; + foreign_physical.devices_in_canonical_order = foreign_devices; + var foreign_mesh = platform.replicated_mesh.*; + foreign_mesh.physical = &foreign_physical; + const pinned = fixture.store.view().createReplicatedTensor("second", null); + fixture.store.view().setMemory(pinned, .host_pinned); + var foreign = [_]Tensor{ fixture.value, pinned }; + for (&foreign) |*tensor| tensor._shape._sharding.mesh = &foreign_mesh; + + // No binding, an empty source, a mesh of other devices, a host kind the + // platform cannot take, a broadcast's strided runs and a program. + const Case = struct { tensor: Tensor, skip: bool = false }; + const cases = [_]Case{ + .{ .tensor = Tensor.fromShape(fixture.value.shape()) }, + .{ .tensor = fixture.empty }, + .{ .tensor = foreign[0] }, + .{ .tensor = foreign[1] }, + .{ .tensor = pinned, .skip = backend.supportsMemory(platform, .host_pinned) }, + .{ .tensor = fixture.store.view().source("value", .{.x}).broad(Shape.init(.{ .y = 2, .x = 4 }, .u8)).withPartitioning(.replicated, .{}).bind() }, + .{ .tensor = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }, + }; + for (cases) |case| { + if (case.skip) continue; + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var output: Buffer = undefined; + try std.testing.expectError(error.InvalidBinding, loader.start(&case.tensor, &output, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + // `start` emptied the buffer before it failed. + try std.testing.expectEqual(0, output._shards.len); + output.deinit(); + try std.testing.expectError(error.InvalidBinding, loader.start(&fixture.value, &output, &fixture.store, .none)); + } +} + +test "buffers of another structure fail start before any buffer is written" { + var fixture: LoaderTestFixture = undefined; + try fixture.init(std.testing.allocator, std.testing.io); + defer fixture.deinit(); + const value = fixture.value; + const second = fixture.second; + + // Each pair holds as many tensors on both sides, in another structure, + // and a pair precedes each mismatch. + const Optionals = struct { head: Tensor, a: ?Tensor, b: ?Tensor }; + try expectStructureMismatch(&fixture, Optionals, .{ .head = value, .a = value, .b = null }, .{ .head = value, .a = null, .b = value }); + const Arms = struct { head: Tensor, arm: union(enum) { a: Tensor, b: Tensor }, last: Tensor }; + try expectStructureMismatch(&fixture, Arms, .{ .head = value, .arm = .{ .a = value }, .last = second }, .{ .head = value, .arm = .{ .b = value }, .last = second }); + const Slices = struct { head: Tensor, first: []Tensor, second: []Tensor }; + var two = [_]Tensor{ value, second }; + var one = [_]Tensor{value}; + var other_one = [_]Tensor{second}; + var other_two = [_]Tensor{ second, value }; + try expectStructureMismatch(&fixture, Slices, .{ .head = value, .first = &two, .second = &one }, .{ .head = value, .first = &other_one, .second = &other_two }); +} + +/// Starts `model` into the buffers bufferized from `built`, which has as +/// many tensors in another structure: `InvalidBinding`, and every buffer as +/// it was. +fn expectStructureMismatch(fixture: *LoaderTestFixture, comptime Model: type, built: Model, model: Model) !void { + const allocator = std.testing.allocator; + var loader = try Loader.init(allocator, std.testing.io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var buffers = try mem.bufferize(allocator, &built); + defer mem.deinitBufferized(allocator, &buffers); + // No tensor has this shape, which `start` would replace with its + // tensor's when it empties a buffer. + const untouched: Shape = .init(.{ 1, 2, 3 }, .u8); + meta.forEachVisit(&buffers, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, shape: Shape) void { + buffer._shape = shape; + } + }.call, .{untouched}); + try std.testing.expectError(error.InvalidBinding, loader.start(&model, &buffers, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + var written: usize = 0; + meta.forEachVisit(&buffers, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, shape: Shape, count: *usize) void { + if (!buffer._shape.eql(shape) or buffer._shards.len != 0) count.* += 1; + } + }.call, .{ untouched, &written }); + try std.testing.expectEqual(0, written); +} From 602a027817b52ce3739fb3ee9a641b27795367c9 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:15:13 +0100 Subject: [PATCH 058/103] zml/testing: add storeOf and load over the new store and loader Two helpers for tests of the new loader. storeOf builds a store over a copy of a registry, since a store takes its registry's entries and a test may want several stores over one checkpoint. load bufferizes a model, loads it from a store with a loader of its own and returns the buffers, for the caller to free with mem.deinitBufferized. zml does not export the new loader, so testing.zig imports its files by path. The loader's tests gain one that starts loads from two stores before one wait. Co-Authored-By: Claude Opus 5.5 --- zml/io/loader.zig | 23 ++++++++++++++++++ zml/testing.zig | 59 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/zml/io/loader.zig b/zml/io/loader.zig index 1740f6d6..8fdd5208 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -561,6 +561,29 @@ test "start publishes the model at once and wait retires it" { try std.testing.expectEqual(null, loader.stats.started_at); } +test "one loader takes loads from different stores before one wait" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var other_store = try testing.storeOf(allocator, &fixture.store.registry, .{ .meshes = &.{fixture.platform.replicated_mesh} }); + defer other_store.deinit(); + const other = other_store.view().createReplicatedTensor("second", null); + + var first: Buffer = undefined; + var second: Buffer = undefined; + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&fixture.value, &first, &fixture.store, .none); + try loader.start(&other, &second, &other_store, .none); + try loader.wait(); + defer first.deinit(); + defer second.deinit(); + try testing.expectShards(allocator, io, &first, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &second, &LoaderTestFixture.second_contents, .default); +} + test "cancel gives up on everything started" { const allocator = std.testing.allocator; const io = std.testing.io; diff --git a/zml/testing.zig b/zml/testing.zig index 023f937a..7748676a 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -7,8 +7,10 @@ const Platform = @import("platform.zig").Platform; const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; const zml = @import("zml.zig"); -/// The new loader's delivery kinds; zml does not export its store. -const LoadMemory = @import("io/TensorStore.zig").Memory; +/// The new loader and its store, which zml does not export. +const Loader = @import("io/loader.zig").Loader; +const LoaderStore = @import("io/TensorStore.zig"); +const LoadMemory = LoaderStore.Memory; const log = std.log.scoped(.@"zml/testing"); @@ -477,6 +479,35 @@ pub fn pattern(bytes: []u8, seed: u8) void { for (bytes, 0..) |*byte, index| byte.* = (@as(u8, @truncate(index *% 29)) +% seed *% 53) | 1; } +/// A store of the new loader over a copy of `registry`, for a test that +/// builds several stores over one checkpoint: a store takes its registry's +/// entries. +pub fn storeOf(allocator: std.mem.Allocator, registry: *const zml.safetensors.TensorRegistry, opts: LoaderStore.Options) !LoaderStore { + var copy: zml.safetensors.TensorRegistry = .init(allocator); + defer copy.deinit(); + for (registry.tensors.values()) |tensor| try copy.registerTensor(tensor); + return .init(allocator, ©, opts); +} + +/// Loads `model` from `store` with the new loader, one of its own, and +/// returns its buffers, which the caller frees with `mem.deinitBufferized`. +/// A recipe for tests: a model's code keeps one loader for all its loads. +pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + model: anytype, + store: *const LoaderStore, +) !zml.Bufferized(@TypeOf(model.*)) { + var buffers = try zml.mem.bufferize(allocator, model); + errdefer zml.mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{}); + defer loader.deinit(); + try loader.start(model, &buffers, store, .none); + try loader.wait(); + return buffers; +} + /// Reads every shard of `buffer` back, in its mesh's device order, and /// checks that it lives in `memory`'s space and holds that device's box of /// `global`, the whole tensor's row-major bytes (sub-byte elements packed). @@ -669,3 +700,27 @@ fn center(slice: anytype, i: usize) @TypeOf(slice) { pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { return std.testing.expectEqual(expected, actual); } + +test load { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: zml.safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var bytes: [8]u8 = undefined; + pattern(&bytes, 1); + try writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "weight", .dtype = "U8", .dims = &.{8}, .bytes = &bytes }, + }); + var store = try storeOf(allocator, ®istry, .{}); + defer store.deinit(); + // The store took a copy: the registry still holds its entries. + try std.testing.expectEqual(1, registry.tensors.count()); + + const Model = struct { weight: zml.Tensor }; + const model: Model = .{ .weight = store.view().createReplicatedTensor("weight", null) }; + var buffers = try load(allocator, io, env(), &model, &store); + defer zml.mem.deinitBufferized(allocator, &buffers); + try expectShards(allocator, io, &buffers.weight, &bytes, .default); +} From 6857537d409772091f6a7198a623f3080a45c5ef Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:17:42 +0100 Subject: [PATCH 059/103] zml/io: run Loader.init's setup on a task Loader.init now checks its options and returns at once. A task of the loader's own warms the device allocators and then creates the backend, so a caller that creates its loader before building the model overlaps that work with its own. When io cannot run the task concurrently, init runs it in place. The first start waits for the task and fails the loader with its error; deinit cancels a set-up no start waited for and destroys what it built. Platform.warmupDeviceAllocators forces lazy device allocators to reserve their pools: a preallocated BFC arena materializes only on its first allocation. It creates and frees a one-byte buffer per device, the devices warmed concurrently or in place when io cannot spawn a task, and returns the first error. Co-Authored-By: Claude Opus 5.5 --- zml/io/loader.zig | 120 ++++++++++++++++++++++++++++++++++++++++------ zml/platform.zig | 47 ++++++++++++++++++ zml/testing.zig | 3 +- 3 files changed, 155 insertions(+), 15 deletions(-) diff --git a/zml/io/loader.zig b/zml/io/loader.zig index 8fdd5208..d842c16d 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -35,7 +35,8 @@ const TensorStore = @import("TensorStore.zig"); const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; -/// Loads models from their checkpoint sources. +/// Loads models from their checkpoint sources. A value: `init` returns at +/// once. /// /// What it borrows. The platform outlives the loader and every buffer it /// loads. `io` outlives the loader: the backend's tasks run on it and open @@ -69,7 +70,12 @@ pub const Loader = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, - backend: Backend, + /// `init`'s task, which warms the device allocators and creates the + /// backend, until the first `start` awaits it. + setup: ?std.Io.Future(anyerror!Backend), + /// What `setup` created; null before the first `start`, and after a + /// failed set-up. + backend: ?Backend = null, /// Submissions published and not yet retired, oldest first. pending: std.Deque(PendingSubmission) = .empty, /// The first error; every later call returns it. @@ -86,23 +92,45 @@ pub const Loader = struct { /// Concurrent source reads at most (`Options.read_parallelism`). pub const max_read_parallelism: usize = 128; - /// Checks `opts` and creates the backend. A read width out of range is - /// `InvalidOptions`, logged. + /// Checks `opts` and returns at once. Warming the device allocators and + /// creating the backend run on a task of the loader's own, or here when + /// `io` has no concurrency; the first `start` waits for that task and + /// returns its error. A read width out of range is `InvalidOptions`, + /// logged. pub fn init( allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options, - ) !Loader { + ) error{InvalidOptions}!Loader { const backend_opts = try backendOptions(opts); + const setup_args = .{ allocator, io, platform, backend_opts }; return .{ .allocator = allocator, .io = io, .platform = platform, - .backend = try .init(allocator, io, platform, backend_opts), + .setup = io.concurrent(setUp, setup_args) catch io.async(setUp, setup_args), }; } + /// `init`'s task: the device allocators warmed, then the backend. + fn setUp(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: backend.Options) anyerror!Backend { + platform.warmupDeviceAllocators(io) catch |err| { + load_log.failure(err, "warm the device allocators", .{}); + return err; + }; + return Backend.init(allocator, io, platform, opts); + } + + /// Waits for `init`'s task the first time: its backend, or its error, + /// for the caller to fail the loader with. + fn awaitSetup(self: *Loader) !void { + const setup = if (self.setup) |*task| task else return; + const result = setup.await(self.io); + self.setup = null; + self.backend = try result; + } + /// Gives up on everything started: aborts the backend (queued reads are /// dropped, reads in flight finish) and retires the pending /// submissions. Each buffer of the started loads is then empty or holds @@ -115,13 +143,17 @@ pub const Loader = struct { if (self.loadsStarted()) self.fail(error.Canceled); } - /// Destroys the backend: its workers and its host staging. The buffers - /// it loaded stay the caller's. Panics on loads neither waited for nor - /// canceled. + /// Cancels `init`'s task when no `start` awaited it, destroying what it + /// built, and destroys the backend: its workers and its host staging. + /// The buffers it loaded stay the caller's. Panics on loads neither + /// waited for nor canceled. pub fn deinit(self: *Loader) void { if (self.loadsStarted()) std.debug.panic("Loader.deinit with loads started, whose store and buffers may be freed already: add `errdefer loader.cancel();` after `start`", .{}); self.pending.deinit(self.allocator); - self.backend.destroy(); + if (self.setup) |*setup| { + if (setup.cancel(self.io)) |backend_| backend_.destroy() else |_| {} + } + if (self.backend) |backend_| backend_.destroy(); self.* = undefined; } @@ -159,7 +191,8 @@ pub const Loader = struct { /// `progress` before then. `start` adds one item to `progress` for each /// tensor, a composite one once; the backend completes them. /// Several loads may be started before one `wait`, from one store or - /// several. + /// several. The first `start` waits for `init`'s task, and fails with + /// its error. pub fn start( self: *Loader, model: anytype, @@ -181,6 +214,8 @@ pub const Loader = struct { store: *const TensorStore, progress: std.Progress.Node, ) !void { + try self.awaitSetup(); + // The load's time starts once the set-up is done. self.stats.begin(self.io); var prepared = try prepareModel(self.allocator, self.platform, store, model, buffers); errdefer prepared.arena.deinit(); @@ -214,7 +249,7 @@ pub const Loader = struct { fn fail(self: *Loader, err: anyerror) void { if (self.failure == null) self.failure = err; // Nothing still pending can succeed. - if (self.pending.len != 0) self.backend.abort(); + if (self.pending.len != 0) self.backend.?.abort(); while (self.pending.len != 0) self.retireOldest() catch {}; } @@ -232,7 +267,7 @@ pub const Loader = struct { var logical_bytes: usize = 0; for (specs) |spec| logical_bytes += spec.shape.byteSize(); try self.pending.ensureUnusedCapacity(self.allocator, 1); - const submission = try self.backend.submit(specs, progress); + const submission = try self.backend.?.submit(specs, progress); self.pending.pushBackAssumeCapacity(.{ .submission = submission, .arena = arena, @@ -246,7 +281,7 @@ pub const Loader = struct { fn retireOldest(self: *Loader) !void { const oldest = self.pending.popFront().?; defer oldest.arena.deinit(); - try self.backend.await(oldest.submission); + try self.backend.?.await(oldest.submission); self.stats.bytes_loaded += oldest.logical_bytes; } }; @@ -673,6 +708,63 @@ test "init checks every option and returns InvalidOptions" { return error.TestUnexpectedResult; } else |err| try std.testing.expectEqual(error.InvalidOptions, err); } + // Nothing started, so `deinit` cancels the set-up or destroys what it + // built. + var loader = try Loader.init(allocator, io, platform, .{}); + loader.deinit(); +} + +test "a failed set-up fails the first start, and deinit frees the rest" { + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(std.testing.allocator, io); + defer fixture.deinit(); + // `init` allocates nothing; the set-up's first allocation, inside the + // backend, fails. + var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 0 }); + var loader = try Loader.init(failing.allocator(), io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var output: Buffer = undefined; + try std.testing.expectError(error.OutOfMemory, loader.start(&fixture.value, &output, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.OutOfMemory); + try std.testing.expect(loader.backend == null); + try std.testing.expectError(error.OutOfMemory, loader.wait()); +} + +test "without concurrency init sets up in place" { + const allocator = std.testing.allocator; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, std.testing.io); + defer fixture.deinit(); + // The testing io, but `io.concurrent` refuses and `io.async` runs the + // function in place. + var vtable = std.testing.io.vtable.*; + vtable.concurrent = struct { + fn concurrent(_: ?*anyopaque, _: usize, _: std.mem.Alignment, _: []const u8, _: std.mem.Alignment, _: *const fn (*const anyopaque, *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { + return error.ConcurrencyUnavailable; + } + }.concurrent; + vtable.async = struct { + fn async(_: ?*anyopaque, result: []u8, _: std.mem.Alignment, context: []const u8, _: std.mem.Alignment, start: *const fn (*const anyopaque, *anyopaque) void) ?*std.Io.AnyFuture { + start(context.ptr, result.ptr); + return null; + } + }.async; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try std.testing.expect(loader.setup.?.any_future == null); + // Nothing started: neither has anything to do. + loader.cancel(); + try loader.wait(); + try std.testing.expectEqual(null, loader.failure); + + var output: Buffer = undefined; + try loader.start(&fixture.value, &output, &fixture.store, .none); + try loader.wait(); + defer output.deinit(); + try testing.expectShards(allocator, io, &output, &LoaderTestFixture.contents, .default); } test "a rejected binding fails the loader" { diff --git a/zml/platform.zig b/zml/platform.zig index f2401715..b2bd3a4e 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -427,6 +427,46 @@ pub const Platform = struct { } else error.Unavailable; } + /// Forces lazy device allocators to materialize their backing pools. + /// + /// In particular, a preallocated GPU BFC allocator does not reserve its + /// fixed arena until its first non-empty allocation. The temporary buffer + /// is released immediately; caching allocators retain their initialized + /// pools for subsequent allocations. Calling this more than once is safe. + /// Devices are warmed concurrently: a serial pass over eight GPUs costs + /// measurable milliseconds ahead of a load. A device whose task `io` + /// cannot spawn is warmed in place. + pub fn warmupDeviceAllocators(self: *const Platform, io: std.Io) !void { + const Worker = struct { + platform: *const Platform, + device_index: usize, + first_error: *stdx.FirstError(pjrt.ApiError), + + fn run(worker: @This()) void { + const dims: []const i64 = &.{1}; + const memory = worker.platform.devices[worker.device_index].memory(.default).?; + const buffer = worker.platform.pjrt_client.createUninitializedBuffer(worker.platform.pjrt_api, .{ + .dims = dims, + .element_type = pjrtx.bufferTypeFromDtype(.u8), + .layout = worker.platform.defaultMemoryLayout(dims, .u8), + .dst = .{ .memory = memory.pjrt_memory }, + }) catch |err| { + _ = worker.first_error.set(err); + return; + }; + buffer.deinit(worker.platform.pjrt_api); + } + }; + var first_error: stdx.FirstError(pjrt.ApiError) = .{}; + var group: std.Io.Group = .init; + for (self.devices, 0..) |_, device_index| { + const worker: Worker = .{ .platform = self, .device_index = device_index, .first_error = &first_error }; + group.concurrent(io, Worker.run, .{worker}) catch worker.run(); + } + try group.await(io); + if (first_error.get()) |err| return err; + } + pub fn formatWithAttributes(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { const tee = "├─ "; const line = "│ "; @@ -1075,3 +1115,10 @@ test "platform defaultMemoryLayout is boring" { }); } } + +test "platform device allocators can be warmed repeatedly" { + const platform = zml.testing.env(); + + try platform.warmupDeviceAllocators(std.testing.io); + try platform.warmupDeviceAllocators(std.testing.io); +} diff --git a/zml/testing.zig b/zml/testing.zig index 7748676a..f6e1b142 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -491,7 +491,8 @@ pub fn storeOf(allocator: std.mem.Allocator, registry: *const zml.safetensors.Te /// Loads `model` from `store` with the new loader, one of its own, and /// returns its buffers, which the caller frees with `mem.deinitBufferized`. -/// A recipe for tests: a model's code keeps one loader for all its loads. +/// A recipe for tests: production code creates its loader before building +/// the model, so the loader's set-up runs meanwhile. pub fn load( allocator: std.mem.Allocator, io: std.Io, From be7e712a455b737b257b016dccd3176025a9c860 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:24:34 +0100 Subject: [PATCH 060/103] zml/io: run binding programs in the loader The new loader now computes the tensors a program binds (Source.bindProgram) instead of refusing them. start checks a computed tensor's placement like any other and queues it; the program tensors are cut, in model order, into groups of at most Options.program_group_bytes of outputs (8 GiB), so the inputs of one layer's experts, adjacent in the checkpoint, are read together. Each input lands in an empty shell through its own spec, checked as start checks every spec. Compiles start at start, on io.concurrent tasks, or in place when io cannot run one, so the bulk reads do not wait behind them. Tensors whose programs are equal over the same placed shapes share one compile (Compute.sameExecutable). Once compiled, an executable must return one tensor of its binding's shape, placed alike on every device, or wait fails with InvalidBinding naming the program. wait publishes the groups oldest first. Before each, it waits for the group's compiles, then retires pending submissions until no other program group is pending, so one group is in flight at a time. Retiring a group runs its programs on the caller's task through Exe.call, whose errors fail the loader, then frees every input shell exactly once, donated or not. A failed or canceled loader drops the queued groups and retires the pending ones without running them. The info line wait logs adds the tensors computed, the programs, the groups and the time spent waiting for compiles. A load gives back the device memory it freed. XLA's VMM allocator, the default for CUDA and ROCm platforms, keeps a freed allocation mapped and counted in bytes_in_use until a request of the same size reuses it, a request beyond the device's limit reclaims it, or an executable with remapped command buffers is destroyed, so without this the last groups' inputs and the programs' temporaries would stay held after the load. Platform.releaseFreedDeviceMemory asks each device for one byte beyond its limit through pjrt's new Client.createUninitializedBufferSilent, which logs its error at debug only: the allocator completes the deferred frees of the device's memory, not those of collective or pinned host memory, then fails with ResourceExhausted, as expected. Any other answer is logged once and ends the VMM drain on that platform. The VMM drain does nothing on platforms created with another allocator, which frees at once (BFC on oneAPI and Metal, CPU, Furiosa), and on a device that reports no limit. A GPU test checks that a freed buffer stops counting in bytes_in_use once drained. wait calls it once the programs are destroyed, when a load was started. A failed or canceled loader does not: its caller frees most of the load's memory afterwards, and after a device fault the request could fail with that fault, logging a second line, or block. Through llmd on two GB300, an earlier build of the stack, with this drain patched in at the end of wait, brought the device memory in use after the load to llmd's on monorepo master at 945cf926, to the byte, on MiMo-V2.6-Flash, Qwen3.8-Flash-Next-FP8 and DeepSeek-V4.1-Flash, where the same build without the drain held 2 to 17 GiB more per device. That drain took at most 63 ms per call for both devices (76 ms for the same request made in llmd right after the load). On ROCm the drain also makes the HIP runtime give back the device memory it keeps outside the allocator, which the plugin's memory stats never count: chiefly the scratch memory of kernels that spill registers, such as XLA's autotuning candidates. ROCr keeps that scratch bound to its queue, up to 3 GiB per XCC by default, and keeps what a queue gives up mapped in its scratch cache, until the driver refuses one of its allocations. releaseRuntimeDeviceMemory in platforms/rocm therefore asks each device, through the libamdhip64.so.7 the plugin loaded, for its whole memory rounded down to 2 MiB: the driver refuses, ROCr reclaims every queue's scratch and frees its caches, and hipMalloc fails with hipErrorOutOfMemory, as expected. This runs whatever the allocator, before the VMM drain, and makes running work on the device wait while ROCr suspends and resumes each queue holding scratch. A missing HIP runtime, or a device that does not refuse, is logged once as a warning and ends it for the process. zml depends on //platforms/rocm directly for this. A GPU test checks that every device refuses and still moves data afterwards; the release itself shows only in the driver's counters. Through llmd on MI300X, an earlier build of the stack's last commit ran interleaved with one without this HIP release (its VMM drain in place). On two GPUs, MiMo-V2.6-Flash's autotuning left 7.4 to 21.3 GiB of it at ready on the GPU that ran it without the release (4 of 4 processes, on both GPUs in one); with it both GPUs were within 238 MiB of llmd on master (4 of 4), the autotuning GPU, 25.5 to 28.4 GiB above the other at the end of the compiles in the 3 processes where only one GPU autotuned, back at the other's level by the time llmd listened. On four GPUs, LFM3-1.7B-FP8's 2.5 to 7.7 GiB on three of them was gone likewise. Start-up time did not grow measurably, and decode speed and answers were unchanged. A probe making the same HIP calls in llmd's process timed each call at 3.3 to 4.5 ms on a GPU whose queue held scratch and under 0.5 ms on one with nothing to free. Co-Authored-By: Claude Opus 5.5 --- pjrt/pjrt.zig | 13 +- platforms/rocm/rocm.zig | 138 ++++++- zml/BUILD.bazel | 5 + zml/io/loader.zig | 772 ++++++++++++++++++++++++++++++++++------ zml/platform.zig | 145 ++++++++ 5 files changed, 963 insertions(+), 110 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index e09073b2..499e753c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -672,8 +672,19 @@ pub const Client = opaque { }; pub fn createUninitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { + return self.createUninitializedBufferThrough(api, args, Api.call); + } + + /// As `createUninitializedBuffer`, for a request the caller expects to + /// fail: its error is logged at debug level only. + pub fn createUninitializedBufferSilent(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { + return self.createUninitializedBufferThrough(api, args, Api.callSilent); + } + + /// `call` is `Api.call` or `Api.callSilent`. + fn createUninitializedBufferThrough(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs, comptime call: anytype) ApiError!*Buffer { var layout = args.layout.toCStruct(); - const ret = try api.call(.PJRT_Client_CreateUninitializedBuffer, .{ + const ret = try call(api, .PJRT_Client_CreateUninitializedBuffer, .{ .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = args.dims.len, diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index ee31bd83..6fcda16b 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -74,6 +74,142 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_rocm.so" }); - break :blk .loadFrom(lib_path); + const api = try pjrt.Api.loadFrom(lib_path); + loaded_hip = Hip.loaded(); + break :blk api; }; } + +/// Whether `releaseRuntimeDeviceMemory` asks the devices for their whole +/// memory. The first call that finds the HIP runtime missing, or a device +/// that does not refuse, logs it and clears this for good. It lives here, +/// not with a PJRT client: the HIP runtime the plugin loaded, and the +/// memory it keeps, belong to the process and serve every client. +var releases_runtime_memory: std.atomic.Value(bool) = .init(true); +/// The HIP calls of `releaseRuntimeDeviceMemory`, resolved by `load`, which +/// zml calls once per process; null when `libamdhip64.so.7` is not loaded +/// or lacks one of them. +var loaded_hip: ?Hip = null; + +/// Makes the HIP runtime the plugin loaded give back the device memory it +/// keeps outside the plugin's allocator, chiefly the scratch of kernels +/// that spill registers, which ROCr keeps bound to its queues and in its +/// caches until the driver refuses one of its allocations. It asks every +/// device in turn for its whole memory, rounded down to 2 MiB: the driver +/// refuses, ROCr reclaims the scratch of the device's queues and frees its +/// caches, and HIP answers `hipErrorOutOfMemory`. Kernels that spill later +/// map their scratch again. Every queue in the process that holds scratch +/// on the device is suspended and resumed, and the call waits for its +/// kernels using scratch while holding the device's ROCr memory lock: +/// running work and allocations on that device stall meanwhile, a stall, +/// not a deadlock. Restores the thread's HIP device, and clears any HIP +/// error left on the thread, earlier ones included. +/// +/// Returns whether every device refused. A missing `libamdhip64.so.7` or +/// symbol, or a device that does not refuse (a grant is freed at once), is +/// logged once and ends the trim for good: the remaining devices are not +/// asked, nor any device in later calls. +pub fn releaseRuntimeDeviceMemory(hip_ordinals: []const c_int) bool { + if (comptime !isEnabled() or builtin.os.tag != .linux) return false; + if (!releases_runtime_memory.load(.monotonic)) return false; + const hip = loaded_hip orelse { + if (releases_runtime_memory.swap(false, .monotonic)) { + log.warn("The HIP runtime (libamdhip64.so.7) is not loaded or lacks a symbol; the device memory it keeps is no longer given back", .{}); + } + return false; + }; + + var previous: c_int = 0; + const has_previous = hip.hipGetDevice(&previous) == hip_success; + defer { + if (has_previous) _ = hip.hipSetDevice(previous); + _ = hip.hipGetLastError(); + } + + for (hip_ordinals) |ordinal| { + const answer = hip.requestWholeDevice(ordinal); + if (answer == .refused) continue; + if (releases_runtime_memory.swap(false, .monotonic)) { + log.warn("HIP device {d} did not refuse a request for its whole memory: {f}; the device memory the HIP runtime keeps is no longer given back", .{ ordinal, answer }); + } + return false; + } + return true; +} + +/// `hipError_t` values (hip_runtime_api.h). +const hip_success: c_int = 0; +const hip_error_out_of_memory: c_int = 2; + +/// The HIP calls `releaseRuntimeDeviceMemory` makes, from the HIP runtime +/// already in the process. +const Hip = struct { + hipGetDevice: *const fn (device: *c_int) callconv(.c) c_int, + hipSetDevice: *const fn (device: c_int) callconv(.c) c_int, + hipMemGetInfo: *const fn (free: *usize, total: *usize) callconv(.c) c_int, + hipMalloc: *const fn (ptr: *?*anyopaque, size: usize) callconv(.c) c_int, + hipFree: *const fn (ptr: ?*anyopaque) callconv(.c) c_int, + hipGetLastError: *const fn () callconv(.c) c_int, + + /// Only finds the library: it never loads it, nor unloads it (see the + /// note on thread-local storage in `load`). `libzml_rocm.so` needs + /// `libamdhip64.so.7`, and the plugin's own dlopen of `libamdhip64.so` + /// resolves to the same file (`zmlxrocm_dlopen`). + fn loaded() ?Hip { + var lib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen("libamdhip64.so.7", .{ .LAZY = true, .NOLOAD = true }) orelse return null, + } }; + var hip: Hip = undefined; + inline for (@typeInfo(Hip).@"struct".field_names) |name| { + @field(hip, name) = lib.lookup(@FieldType(Hip, name), name) orelse return null; + } + return hip; + } + + /// Asks HIP for the whole memory of the device, made the current one, + /// when less than that is free, so that the driver cannot grant it. + /// Frees what HIP grants anyway. + fn requestWholeDevice(hip: Hip, ordinal: c_int) Answer { + const set_device = hip.hipSetDevice(ordinal); + if (set_device != hip_success) return .{ .set_device = set_device }; + var free: usize = 0; + var total: usize = 0; + const mem_get_info = hip.hipMemGetInfo(&free, &total); + if (mem_get_info != hip_success) return .{ .mem_get_info = mem_get_info }; + // ROCr rounds a request up to 2 MiB and refuses one beyond the + // device's size itself, without the driver's refusal that trims. + const request = std.mem.alignBackward(usize, total, 2 << 20); + if (free >= request) return .{ .not_asked = .{ .free = free, .request = request } }; + var ptr: ?*anyopaque = null; + const answer = hip.hipMalloc(&ptr, request); + if (answer == hip_error_out_of_memory) return .refused; + if (answer == hip_success) _ = hip.hipFree(ptr); + return .{ .malloc = answer }; + } +}; + +/// How a device answered `Hip.requestWholeDevice`. +const Answer = union(enum) { + /// hipMalloc answered `hipErrorOutOfMemory`, as expected. + refused, + /// So much is free that the driver could grant the request. + not_asked: struct { free: usize, request: usize }, + /// hipMalloc's other answers: `hip_success` for a grant, freed at once. + malloc: c_int, + /// The `hipError_t` of hipSetDevice or hipMemGetInfo, which failed. + set_device: c_int, + mem_get_info: c_int, + + pub fn format(answer: Answer, writer: *std.Io.Writer) std.Io.Writer.Error!void { + switch (answer) { + .refused => try writer.writeAll("refused"), + .not_asked => |memory| try writer.print("not asked, {d} bytes being free for a request of {d}", .{ memory.free, memory.request }), + .malloc => |code| if (code == hip_success) + try writer.writeAll("granted, then freed") + else + try writer.print("hipMalloc returned hipError_t {d}", .{code}), + .set_device => |code| try writer.print("hipSetDevice returned hipError_t {d}", .{code}), + .mem_get_info => |code| try writer.print("hipMemGetInfo returned hipError_t {d}", .{code}), + } + } +}; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index b889c8fc..4c605a30 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -192,6 +192,11 @@ zig_library( "//platforms/cuda/flashinfer_cutlass_moe", ], "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": [ + "//platforms/rocm", + ], + "//conditions:default": [], }), ) diff --git a/zml/io/loader.zig b/zml/io/loader.zig index d842c16d..8bcfd1d7 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -1,21 +1,35 @@ -//! Checkpoint loading. `Loader.start` takes a whole model and publishes -//! every tensor it fills as one submission; `wait` retires it. Backends -//! only read sources into buffers; lookup and the checks stay in this -//! shared front end. The contract is in the pub doc comments below. +//! Checkpoint loading. `Loader.start` takes a whole model: it starts +//! compiling the programs that compute some of its tensors +//! (`Source.bindProgram`) and publishes every tensor the loader fills +//! itself. `wait` then publishes the programs' inputs in groups, runs each +//! group's programs on the caller's task once its inputs landed, and +//! retires everything. Backends only read sources into buffers; lookup, +//! compilation, admission and execution stay in this shared front end. The +//! contract is in the pub doc comments below. //! -//! From bindings to specs. `start` checks every tensor once, before -//! anything is published (`prepareSpec`, `checkPlacement`). It lowers each -//! bound source to bytes (`Source.Placed.lower`, the one rule), one part -//! per piece: a part of the destination's logical bytes (the row-major -//! bytes of its shape, before the sharding splits them), a run of `width` -//! bytes repeated over strided dimensions. A whole checkpoint tensor is one -//! part. Each `LoadSpec` is one destination, which the backends trust as -//! `backend.LoadSpec` states. Only `Source` makes bindings and its pieces -//! are disjoint by construction: nothing here checks that (Source's tests -//! do). +//! From bindings to specs. `start` checks every tensor and program input +//! once, before anything is published (`prepareSpec`, `checkPlacement`). +//! It lowers each bound source to bytes (`Source.Placed.lower`, the one +//! rule), one part per piece: a part of the destination's logical bytes +//! (the row-major bytes of its shape, before the sharding splits them), a +//! run of `width` bytes repeated over strided dimensions. A whole +//! checkpoint tensor is one part. Each `LoadSpec` is one destination, which +//! the backends trust as `backend.LoadSpec` states. Only `Source` makes +//! bindings and its pieces are disjoint by construction: nothing here +//! checks that (Source's tests do). +//! +//! Order and admission. The tensors the loader fills go out first, as one +//! submission. A group holds the tensors of consecutive programs in model +//! order, up to `program_group_bytes` of outputs, so the inputs of one +//! layer's experts, adjacent in the checkpoint, are read together. Before +//! publishing a group, `wait` waits for its programs to compile, then +//! retires the oldest pending submissions until no program group is +//! pending: one program group is in flight at a time. const std = @import("std"); const Buffer = @import("../buffer.zig").Buffer; +const Compiler = @import("../Compiler.zig"); +const Exe = @import("../exe.zig").Exe; const mem = @import("../mem.zig"); const Bufferized = mem.Bufferized; const meta = @import("../meta.zig"); @@ -46,8 +60,9 @@ const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; /// the loader. /// /// Start and wait serially on the owning task; source reads and transfers -/// run concurrently inside the backend. Submissions are retired in publish -/// order. +/// run concurrently inside the backend, compiles on tasks of their own, or +/// inline when `io` cannot run one concurrently. Submissions are retired in +/// publish order. /// /// What fails. Every error of `start` and `wait` fails the loader: /// everything started is canceled (`cancel`), and every later call returns @@ -56,16 +71,17 @@ const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; /// `Canceled`, the caller's own `cancel`, which are returned unlogged. A /// model whose bindings are wrong is `InvalidBinding`, logged with the /// tensor's field path, the key its source reads (none for a source that -/// reads nothing or a run inside a byte) and the rule it breaks; buffers -/// shaped unlike the model (`mem.pairBufferized`) too, logged with the -/// field path where they differ. The other errors a caller can act on: a -/// missing checkpoint file is `FileNotFound`; a denied one `AccessDenied` -/// locally and `PermissionDenied` remotely; a file that ends inside a -/// tensor `UnexpectedEndOfFile`, because it changed after its header was -/// read; host memory the backend cannot allocate `OutOfMemory`; device -/// memory that runs out PJRT's `ResourceExhausted`; `cancel` `Canceled`. -/// The rest (PJRT, a remote that failed) passes through as its source -/// names it. +/// reads nothing or a run inside a byte) and the rule it breaks, or for a +/// program whose output differs, the program's name and the shape it +/// computes; buffers shaped unlike the model (`mem.pairBufferized`) too, +/// logged with the field path where they differ. The other errors a caller +/// can act on: a missing checkpoint file is `FileNotFound`; a denied one +/// `AccessDenied` locally and `PermissionDenied` remotely; a file that ends +/// inside a tensor `UnexpectedEndOfFile`, because it changed after its +/// header was read; host memory the backend cannot allocate `OutOfMemory`; +/// device memory that runs out PJRT's `ResourceExhausted`; `cancel` +/// `Canceled`. The rest (PJRT, the compiler, a remote that failed) passes +/// through as its source names it. pub const Loader = struct { allocator: std.mem.Allocator, io: std.Io, @@ -76,6 +92,13 @@ pub const Loader = struct { /// What `setup` created; null before the first `start`, and after a /// failed set-up. backend: ?Backend = null, + /// `Options.program_group_bytes`. + program_group_bytes: usize, + /// The programs of the loads started since the last `wait`, each + /// compiled once. + programs: std.ArrayList(*CompiledProgram) = .empty, + /// Program groups started and not yet published, oldest first. + queued: std.Deque(Group) = .empty, /// Submissions published and not yet retired, oldest first. pending: std.Deque(PendingSubmission) = .empty, /// The first error; every later call returns it. @@ -87,6 +110,10 @@ pub const Loader = struct { /// the load. Local checkpoints read fastest at 12 to 16 on every /// host measured. read_parallelism: usize = 16, + /// Output bytes of the program tensors one group holds at most, one + /// tensor at least. A group's inputs are read together and freed + /// once its programs ran. + program_group_bytes: usize = 8 << 30, }; /// Concurrent source reads at most (`Options.read_parallelism`). @@ -110,6 +137,7 @@ pub const Loader = struct { .io = io, .platform = platform, .setup = io.concurrent(setUp, setup_args) catch io.async(setUp, setup_args), + .program_group_bytes = opts.program_group_bytes, }; } @@ -131,11 +159,12 @@ pub const Loader = struct { self.backend = try result; } - /// Gives up on everything started: aborts the backend (queued reads are - /// dropped, reads in flight finish) and retires the pending - /// submissions. Each buffer of the started loads is then empty or holds - /// its shards, whose contents are unspecified (unwritten, errored, or - /// partly written); read none, and free the tree as after a load + /// Gives up on everything started: drops the queued groups, aborts the + /// backend (queued reads are dropped, reads in flight finish), retires + /// the pending submissions without running their programs, and waits + /// for the compiles. Each buffer of the started loads is then empty or + /// holds its shards, whose contents are unspecified (unwritten, errored, + /// or partly written); read none, and free the tree as after a load /// (`mem.deinitBufferized`). The loader is failed afterwards. For an /// error between `start` and `wait`; nothing to do when nothing was /// started. @@ -149,7 +178,9 @@ pub const Loader = struct { /// waited for nor canceled. pub fn deinit(self: *Loader) void { if (self.loadsStarted()) std.debug.panic("Loader.deinit with loads started, whose store and buffers may be freed already: add `errdefer loader.cancel();` after `start`", .{}); + self.queued.deinit(self.allocator); self.pending.deinit(self.allocator); + self.programs.deinit(self.allocator); if (self.setup) |*setup| { if (setup.cancel(self.io)) |backend_| backend_.destroy() else |_| {} } @@ -159,7 +190,7 @@ pub const Loader = struct { /// Whether loads are started and neither waited for nor canceled. fn loadsStarted(self: *const Loader) bool { - return self.pending.len != 0; + return self.queued.len != 0 or self.pending.len != 0 or self.programs.items.len != 0; } /// Starts loading every tensor of `model` into its buffer in `buffers`, @@ -167,16 +198,18 @@ pub const Loader = struct { /// Each tensor is placed by its shape's sharding (replicated without a /// mesh), into the memory its binding names, assembled from the pieces /// of its source (`Source.Placed.bind`) with every byte no piece covers - /// set to zero. All of these are published here, as one submission. + /// set to zero. All of these are published here, as one submission. A + /// tensor a program computes (`Source.bindProgram`) starts its program + /// compiling, once for every tensor of an equal program over the same + /// placed shapes; `wait` lands its inputs and runs it. /// - /// Every tensor is checked once, before anything is published: a tensor - /// `store` did not bind, a tensor a program computes - /// (`Source.bindProgram`, which this loader does not run), a source - /// that reads no bytes (an empty checkpoint tensor among them), a source - /// that this loader cannot assemble (a run that starts or ends inside a - /// byte, or strided runs: a transpose, a broadcast, an interleave or - /// interior padding), a sub-byte last dimension that does not pack into - /// whole bytes, a placement its mesh does not divide or on devices the + /// Every tensor and program input is checked once, before anything is + /// published: a tensor `store` did not bind, a source that reads no + /// bytes (an empty checkpoint tensor among them), a source that this + /// loader cannot assemble (a run that starts or ends inside a byte, or + /// strided runs: a transpose, a broadcast, an interleave or interior + /// padding), a sub-byte last dimension that does not pack into whole + /// bytes, a placement its mesh does not divide or on devices the /// platform lacks, and a memory kind the platform cannot take are /// `error.InvalidBinding`. /// @@ -189,7 +222,8 @@ pub const Loader = struct { /// to free: `mem.deinitBufferized` frees the tree either way. `buffers` /// and `store` are borrowed until `wait` or `cancel` returns; do not end /// `progress` before then. `start` adds one item to `progress` for each - /// tensor, a composite one once; the backend completes them. + /// tensor the loader fills, a composite one once, and for each program + /// input; the backend completes them. /// Several loads may be started before one `wait`, from one store or /// several. The first `start` waits for `init`'s task, and fails with /// its error. @@ -219,47 +253,168 @@ pub const Loader = struct { self.stats.begin(self.io); var prepared = try prepareModel(self.allocator, self.platform, store, model, buffers); errdefer prepared.arena.deinit(); - progress.increaseEstimatedTotalItems(prepared.specs.len); + var items = prepared.specs.len; + for (prepared.computed) |item| items += item.compute.inputs.len; + progress.increaseEstimatedTotalItems(items); + // Queued first: their programs compile while the rest is read. + try self.queueGroups(store, prepared.computed, progress); if (prepared.specs.len == 0) { prepared.arena.deinit(); return; } - try self.submit(prepared.specs, prepared.arena, progress); + try self.submit(prepared.specs, prepared.arena, &.{}, progress); } - /// Retires every submission in publish order. Returns once every - /// started load is in its buffers, and logs one info line: the bytes - /// loaded, the time since the first `start` and their rate. Idempotent: - /// with nothing started it returns at once and logs nothing. + /// Publishes the queued program groups, oldest first, each once its + /// programs compiled and no other program group is pending (`admit`), + /// and retires every submission in publish order, running a group's + /// programs on this task once its inputs landed. Returns once every + /// started load is in its buffers, with the freed device memory, its + /// programs' inputs and temporaries among it, given back to the devices + /// (`Platform.releaseFreedDeviceMemory`), and logs one info line: the + /// bytes loaded, the time since the first `start` and their rate, then + /// what the programs computed and how long publishing waited for their + /// compiles. A program that fails to compile or run fails it; one that + /// does not return its tensor's shape, placed alike, is + /// `InvalidBinding`. Idempotent: with nothing started it returns at + /// once and logs nothing. pub fn wait(self: *Loader) !void { if (self.failure) |err| return err; + const started = self.loadsStarted(); self.drain() catch |err| { self.fail(err); return err; }; + self.releasePrograms(); + if (started) self.platform.releaseFreedDeviceMemory(); } fn drain(self: *Loader) !void { - while (self.pending.len != 0) try self.retireOldest(); - self.stats.end(self.io); + var groups: usize = 0; + var tensors: usize = 0; + while (self.queued.popFront()) |group| { + tensors += group.calls.len; + try self.publish(group); + groups += 1; + } + while (self.pending.len != 0) try self.retireOldest(true); + self.stats.end(self.io, tensors, self.programs.items.len, groups); } /// Records `err` unless the loader failed already, and gives up on /// everything started (`cancel`). The one writer of `failure`. fn fail(self: *Loader, err: anyerror) void { if (self.failure == null) self.failure = err; + while (self.queued.popFront()) |group| group.deinit(self.allocator); // Nothing still pending can succeed. if (self.pending.len != 0) self.backend.?.abort(); - while (self.pending.len != 0) self.retireOldest() catch {}; + while (self.pending.len != 0) self.retireOldest(false) catch {}; + self.releasePrograms(); } - /// One submission over `specs`, which live in `arena`. Takes `arena` - /// once the submission is published; on failure the caller still owns - /// it. + /// Queues the tensors of `computed`, in model order, in groups of at + /// most `program_group_bytes` of outputs, one tensor at least. + fn queueGroups(self: *Loader, store: *const TensorStore, computed: []const Computed, progress: std.Progress.Node) !void { + var first: usize = 0; + while (first < computed.len) { + var end = first + 1; + var bytes = computed[first].shape.byteSize(); + while (end < computed.len and bytes + computed[end].shape.byteSize() <= self.program_group_bytes) : (end += 1) { + bytes += computed[end].shape.byteSize(); + } + try self.queued.ensureUnusedCapacity(self.allocator, 1); + self.queued.pushBackAssumeCapacity(try self.prepareGroup(store, computed[first..end], progress)); + first = end; + } + } + + /// The group computing `computed`: each tensor's program, the input + /// shells it runs over and the specs that fill them, one per input, + /// checked as `start` checks every spec (`prepareSpec`) before the + /// program's compile starts, which packs their shapes. + fn prepareGroup(self: *Loader, store: *const TensorStore, computed: []const Computed, progress: std.Progress.Node) PrepareError!Group { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + errdefer arena.deinit(); + const calls = try self.allocator.alloc(Call, computed.len); + var prepared: usize = 0; + errdefer { + for (calls[0..prepared]) |*call| call.deinit(self.allocator); + self.allocator.free(calls); + } + var specs: std.ArrayList(LoadSpec) = .empty; + for (computed, calls) |item, *call| { + const inputs = try self.allocator.alloc(Buffer, item.compute.inputs.len); + errdefer self.allocator.free(inputs); + for (item.compute.inputs, inputs, 0..) |source, *input, index| { + input.* = .{ ._platform = self.platform, ._shape = source.shape, ._shards = .empty }; + const target: Target = .{ .path = item.path, .shape = item.shape, .input = .{ .index = index, .shape = source.shape } }; + const spec = try prepareSpec(arena.allocator(), self.platform, target, source, .default, input); + try specs.append(arena.allocator(), spec); + } + const compiled = try self.compiledProgram(store, item.compute, item.shape); + call.* = .{ .path = try arena.allocator().dupe(u8, item.path), .program = compiled, .inputs = inputs, .output = item.output }; + prepared += 1; + } + return .{ .calls = calls, .specs = specs.items, .arena = arena, .progress = progress }; + } + + /// The compiled program computing a tensor of `output` from `compute`: + /// one started already for an equal program over the same placed + /// shapes, else a compile started now. + fn compiledProgram(self: *Loader, store: *const TensorStore, compute: TensorStore.Binding.Compute, output: Shape) std.mem.Allocator.Error!*CompiledProgram { + for (self.programs.items) |compiled| { + if (compiled.compute.sameExecutable(compiled.output, compute, output)) return compiled; + } + try self.programs.ensureUnusedCapacity(self.allocator, 1); + const compiled = try CompiledProgram.start(self.allocator, self.io, self.platform, store.meshes, compute, output); + self.programs.appendAssumeCapacity(compiled); + return compiled; + } + + fn releasePrograms(self: *Loader) void { + for (self.programs.items) |compiled| compiled.destroy(self.allocator, self.io); + self.programs.clearRetainingCapacity(); + } + + /// Publishes `group` once its programs compiled and no other program + /// group is pending: its calls and its specs' arena move to the pending + /// submission, or are freed when this fails. + fn publish(self: *Loader, group: Group) !void { + errdefer group.deinit(self.allocator); + const started: std.Io.Timestamp = .now(self.io, .awake); + for (group.calls) |call| try call.program.await(self.io, self.platform); + self.stats.compile_wait_ns +|= @intCast(started.untilNow(self.io, .awake).toNanoseconds()); + try self.admit(); + try self.submit(group.specs, group.arena, group.calls, group.progress); + } + + /// Retires pending submissions, oldest first, until no program group is + /// pending, so one program group is in flight at a time. A bulk load + /// published before the pending group is retired with it. + fn admit(self: *Loader) !void { + var retired: usize = 0; + while (self.programGroupPending()) : (retired += 1) try self.retireOldest(true); + self.stats.admission_retires += retired; + } + + /// Whether a program group is pending: only retiring one frees room + /// for the next (a bulk load's buffers stay). + fn programGroupPending(self: *const Loader) bool { + var it = self.pending.iterator(); + while (it.nextPtr()) |submission| { + if (submission.calls.len != 0) return true; + } + return false; + } + + /// One submission over `specs`, which live in `arena`. Takes `arena` and + /// `calls` once the submission is published; on failure the caller still + /// owns them. fn submit( self: *Loader, specs: []const LoadSpec, arena: std.heap.ArenaAllocator, + calls: []Call, progress: std.Progress.Node, ) !void { // One spec per destination, however many parts fill it: its bytes @@ -271,19 +426,35 @@ pub const Loader = struct { self.pending.pushBackAssumeCapacity(.{ .submission = submission, .arena = arena, + .calls = calls, .logical_bytes = logical_bytes, }); self.stats.submissions += 1; } /// Retires the oldest pending submission: waits for its reads and - /// transfers, and counts its bytes once they landed. - fn retireOldest(self: *Loader) !void { - const oldest = self.pending.popFront().?; - defer oldest.arena.deinit(); + /// transfers, runs its programs when `execute`, frees its inputs either + /// way and counts its bytes once they ran. + fn retireOldest(self: *Loader, execute: bool) !void { + var oldest = self.pending.popFront().?; + defer self.release(&oldest); try self.backend.?.await(oldest.submission); + if (!execute) return; + for (oldest.calls) |*call| { + call.execute(self.allocator, self.io) catch |err| { + const target: Target = .{ .path = call.path, .shape = call.program.output }; + load_log.failure(err, "run program {s} computing {f}: inputs={d}", .{ call.program.name(), target, call.inputs.len }); + return err; + }; + } self.stats.bytes_loaded += oldest.logical_bytes; } + + fn release(self: *Loader, oldest: *PendingSubmission) void { + oldest.arena.deinit(); + for (oldest.calls) |*call| call.deinit(self.allocator); + self.allocator.free(oldest.calls); + } }; /// The loader's counters: read only by logs and tests; nothing here @@ -292,12 +463,17 @@ const Stats = struct { /// When the first `start` since the last `wait` ran: `wait` logs the /// load's time from there. started_at: ?std.Io.Timestamp = null, - /// Logical bytes of the submissions retired since that `start`: each - /// destination's shape once, so a composite tensor counts its padding - /// and not its sources one by one. + /// Logical bytes of the submissions retired with execution since that + /// `start`: each destination's shape once, so a composite tensor counts + /// its padding and not its sources one by one, and a program group + /// counts its inputs. bytes_loaded: usize = 0, + /// How long publishing waited for compiles since the last `wait`. + compile_wait_ns: u64 = 0, /// Submissions published since `init`. submissions: usize = 0, + /// Submissions retired to admit a program group, bulk loads included. + admission_retires: usize = 0, /// Times and counts a load from the first `start` since the last /// `wait`; later starts add to it. @@ -308,37 +484,173 @@ const Stats = struct { } /// Logs `wait`'s info line, unless nothing was started, and ends the - /// load's timing. - fn end(self: *Stats, io: std.Io) void { + /// load's timing. `programs` programs computed `tensors` tensors in + /// `groups` groups. + fn end(self: *Stats, io: std.Io, tensors: usize, programs: usize, groups: usize) void { const started_at = self.started_at orelse return; const took = started_at.untilNow(io, .awake); const seconds = @as(f64, @floatFromInt(@max(took.nanoseconds, 1))) / std.time.ns_per_s; const rate: u64 = @intFromFloat(@as(f64, @floatFromInt(self.bytes_loaded)) / seconds); - load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ self.bytes_loaded, took, rate }); + if (groups == 0) { + load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ self.bytes_loaded, took, rate }); + } else { + load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s], computed {d} tensors with {d} programs in {d} groups, {d} ms waiting for compiles", .{ + self.bytes_loaded, + took, + rate, + tensors, + programs, + groups, + self.compile_wait_ns / std.time.ns_per_ms, + }); + } self.started_at = null; + self.compile_wait_ns = 0; } }; -/// One published submission: what to wait for and what its accounting -/// owes. +/// One published submission: what to wait for, what to run afterwards and +/// what its accounting owes. const PendingSubmission = struct { submission: backend.Submission, /// The specs, which the backend reads until the submission is awaited /// (`backend.LoadSpec`). arena: std.heap.ArenaAllocator, + /// Run in order once the reads are done; freed with their inputs when + /// the submission is retired. Empty for a bulk load. + calls: []Call, logical_bytes: usize, }; -/// A model as `start` takes it: one spec per tensor, in `arena` with the -/// specs' parts and the tensors' field paths. +/// Program tensors whose inputs `wait` publishes as one submission. +const Group = struct { + calls: []Call, + /// One per program input, filling the calls' input shells; they and + /// their parts live in `arena`. + specs: []LoadSpec, + arena: std.heap.ArenaAllocator, + progress: std.Progress.Node, + + fn deinit(self: Group, allocator: std.mem.Allocator) void { + for (self.calls) |*call| call.deinit(allocator); + allocator.free(self.calls); + self.arena.deinit(); + } +}; + +/// A program compiling, then compiled. +const CompiledProgram = struct { + compute: TensorStore.Binding.Compute, + output: Shape, + future: std.Io.Future(Compiler.Error!Exe), + /// Set by the first `await`. + exe: ?Exe = null, + + fn start( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + meshes: []const *const Sharding.Mesh, + compute: TensorStore.Binding.Compute, + output: Shape, + ) std.mem.Allocator.Error!*CompiledProgram { + const self = try allocator.create(CompiledProgram); + // Concurrent, as `io.async` may run the compile right here on a busy + // threaded io, ahead of the reads `start` publishes after it. + const args = .{ compute, allocator, io, platform, meshes, output }; + const compile = TensorStore.Binding.Compute.compile; + self.* = .{ + .compute = compute, + .output = output, + .future = io.concurrent(compile, args) catch io.async(compile, args), + }; + return self; + } + + /// Waits for the compile the first time, then checks that the + /// executable computes one tensor of `output`, placed alike + /// (`checkOutput`). + fn await(self: *CompiledProgram, io: std.Io, platform: *const Platform) !void { + if (self.exe != null) return; + self.exe = self.future.await(io) catch |err| { + load_log.failure(err, "compile program {s} computing {f}", .{ self.name(), self.output }); + return err; + }; + try checkOutput(platform, self.name(), self.output, &self.exe.?); + } + + fn name(self: *const CompiledProgram) []const u8 { + return self.compute.program.name(); + } + + fn destroy(self: *CompiledProgram, allocator: std.mem.Allocator, io: std.Io) void { + if (self.exe) |*exe| { + exe.deinit(); + } else if (self.future.cancel(io)) |exe| { + exe.deinit(); + } else |_| {} + allocator.destroy(self); + } +}; + +/// One tensor of a program group, a pending `Exe.call`: its program, the +/// input shells the group's submission fills, and the model's buffer it +/// computes. +const Call = struct { + /// The tensor's field path, in the group's arena: `prepareModel`'s + /// arena may be freed before the call runs. + path: []const u8, + program: *CompiledProgram, + inputs: []Buffer, + output: *Buffer, + + /// Runs the program's executable, awaited when the group was + /// published, over the loaded inputs on the calling task. + /// `Exe.call` leaves every input handle to the caller, donated ones + /// included (their contents are gone), so `deinit` frees them all, + /// exactly once; a runner would destroy the donated handles itself. + /// A failed execution, an input whose transfer failed included, is + /// returned and leaves `output` unwritten. + fn execute(self: *const Call, allocator: std.mem.Allocator, io: std.Io) !void { + const exe = &self.program.exe.?; + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + args.set(.{self.inputs}); + try exe.call(io, args, &results, .{ .wait = true }); + self.output.* = results.get(Buffer); + } + + /// Frees the inputs, donated or not; a shell the loader never wrote owns + /// no shards. + fn deinit(self: *Call, allocator: std.mem.Allocator) void { + for (self.inputs) |*input| input.deinit(); + allocator.free(self.inputs); + } +}; + +/// A tensor a program computes, at `path` in the model, the buffer it +/// computes into, and its program and inputs. +const Computed = struct { + path: []const u8, + shape: Shape, + output: *Buffer, + compute: TensorStore.Binding.Compute, +}; + +/// A model as `start` takes it: one spec per tensor the loader fills, and +/// the tensors programs compute, all in `arena` with the specs' parts and +/// the tensors' field paths. const PreparedModel = struct { arena: std.heap.ArenaAllocator, specs: []const LoadSpec, + computed: []const Computed, }; -/// Pairs each tensor of `model` with its buffer and binding: each tensor -/// takes one spec (`prepareSpec`). A tensor a program computes is -/// `InvalidBinding`: this loader runs no programs. +/// Pairs each tensor of `model` with its buffer and binding: a tensor the +/// loader fills takes one spec (`prepareSpec`), a tensor a program computes +/// is placed like any other (`checkPlacement`) and computed. fn prepareModel( allocator: std.mem.Allocator, platform: *const Platform, @@ -359,11 +671,12 @@ fn prepareModel( }, error.OutOfMemory => return error.OutOfMemory, }; - // Empty until the backend writes it whole, so the caller can free the - // tree whatever fails. + // Empty until a backend or a program writes it whole, so the caller can + // free the tree whatever fails. for (pairs) |pair| pair.buffer.* = .{ ._platform = platform, ._shape = pair.tensor.shape(), ._shards = .empty }; var specs: std.ArrayList(LoadSpec) = try .initCapacity(arena.allocator(), pairs.len); + var computed: std.ArrayList(Computed) = try .initCapacity(arena.allocator(), pairs.len); for (pairs) |pair| { const tensor = pair.tensor; const target: Target = .{ .path = pair.path, .shape = tensor.shape() }; @@ -374,21 +687,30 @@ fn prepareModel( switch (binding) { .load => |load| specs.appendAssumeCapacity(try prepareSpec(arena.allocator(), platform, target, load.source, load.memory, pair.buffer)), .compute => |compute| { - load_log.err("{f} is computed by program {s}, which this loader does not run", .{ target, compute.program.name() }); - return error.InvalidBinding; + try checkPlacement(platform, target, compute.program.name()); + computed.appendAssumeCapacity(.{ .path = pair.path, .shape = tensor.shape(), .output = pair.buffer, .compute = compute }); }, } } - return .{ .arena = arena, .specs = specs.items }; + return .{ .arena = arena, .specs = specs.items, .computed = computed.items }; } /// What a spec fills, as the logs name it: the model's tensor at `path` -/// (`mem.Pair.path`) of `shape`. +/// (`mem.Pair.path`) of `shape`, or `input` of the program computing it. const Target = struct { path: []const u8, shape: Shape, + input: ?Input = null, + + const Input = struct { index: usize, shape: Shape }; + + /// The shape the spec fills: the input's, else the tensor's. + fn destination(self: Target) Shape { + return if (self.input) |input| input.shape else self.shape; + } pub fn format(self: Target, writer: *std.Io.Writer) !void { + if (self.input) |input| try writer.print("program input {d} {f} of ", .{ input.index, input.shape }); try writer.print("model{s} {f}", .{ self.path, self.shape }); } }; @@ -407,7 +729,7 @@ fn prepareSpec( memory: Memory, output: *Buffer, ) PrepareError!LoadSpec { - const shape = target.shape; + const shape = target.destination(); const parts = source.lower(arena) catch |err| switch (err) { error.OutOfMemory => |e| return e, error.SubBytePosition => { @@ -438,12 +760,13 @@ fn prepareSpec( return .{ .shape = shape, .memory = memory, .output = output, .parts = parts }; } -/// Checks that `target`, from `from` (its source's key), places on -/// `platform`'s devices, in any memory space: a sub-byte last dimension -/// packs into whole bytes (`Shape.packedShape`), its mesh partitions divide -/// its dimensions, and every device of its mesh is one of the platform's. +/// Checks that `target`, from `from` (its source's key or its program), +/// places on `platform`'s devices, in any memory space: a sub-byte last +/// dimension packs into whole bytes (`Shape.packedShape`), its mesh +/// partitions divide its dimensions, and every device of its mesh is one of +/// the platform's. fn checkPlacement(platform: *const Platform, target: Target, from: []const u8) error{InvalidBinding}!void { - const shape = target.shape; + const shape = target.destination(); const bits = shape.dtype().bitSizeOf(); if (bits < 8 and (shape.rank() == 0 or @rem(shape.dim(-1), 8 / bits) != 0)) { load_log.err("{f} from {s}: its last dimension does not pack into whole bytes", .{ target, from }); @@ -472,6 +795,44 @@ fn backendOptions(opts: Loader.Options) error{InvalidOptions}!backend.Options { return .{ .read_parallelism = opts.read_parallelism }; } +/// A program must compute one tensor of `expected`, placed on the same +/// devices, each holding the same slice. The comparison is per device, not +/// by mesh pointer and partition, so `.open` and `.replicated`, or a null +/// mesh and the replicated mesh, agree when they place alike. `expected` +/// places on the platform's devices (`checkPlacement`). +fn checkOutput(platform: *const Platform, name: []const u8, expected: Shape, exe: *const Exe) error{InvalidBinding}!void { + if (exe.output_shapes.len != 1 or !expected.eql(exe.output_shapes[0])) { + load_log.err("program {s} computing {f} returns {d} outputs, the first {f}", .{ name, expected, exe.output_shapes.len, if (exe.output_shapes.len != 0) exe.output_shapes[0] else expected }); + return error.InvalidBinding; + } + const actual = exe.output_shapes[0]; + const want_shape = platform.placedShape(expected).packedShape(); + const want = Sharding.Placement.init(&want_shape) catch |err| + std.debug.panic("loader: {f} does not place, though checked: {s}", .{ expected, @errorName(err) }); + const got_shape = platform.placedShape(actual).packedShape(); + const got = Sharding.Placement.init(&got_shape) catch |err| { + load_log.err("program {s}: placement of its output {f}: {s}", .{ name, actual, @errorName(err) }); + return error.InvalidBinding; + }; + const want_devices = want.mesh().?.devicesInCanonicalOrder(); + const got_devices = got.mesh().?.devicesInCanonicalOrder(); + if (want_devices.len != got_devices.len) { + load_log.err("program {s} places its output {f} on {d} devices, the tensor {f} on {d}", .{ name, actual, got_devices.len, expected, want_devices.len }); + return error.InvalidBinding; + } + for (want_devices, got_devices, 0..) |w, g, device_index| { + const want_slices = want.slices(device_index, want_shape); + const got_slices = got.slices(device_index, got_shape); + const same_slices = want_slices.len == got_slices.len and for (want_slices.constSlice(), got_slices.constSlice()) |a, b| { + if (a.start != b.start or a.size != b.size) break false; + } else true; + if (w.id != g.id or !same_slices) { + load_log.err("program {s} places its output {f} unlike the tensor {f} on device {d}", .{ name, actual, expected, w.id }); + return error.InvalidBinding; + } + } +} + /// A four-byte tensor `value`, a four-byte `second`, a `missing` entry in a /// file that does not exist, an `empty` tensor and a CPU platform. Pinned /// after `init`. @@ -552,39 +913,61 @@ const LoaderTestFixture = struct { } }; + /// The program adding `value` to every byte of its one input. + const Add = struct { + value: u8, + + pub fn forward(self: Add, _: Shape, inputs: []const Tensor) Tensor { + return inputs[0].add(Tensor.scalar(self.value, .u8)); + } + }; + /// Whether `loader` failed with `expected` and holds nothing started. fn expectFailed(loader: *const Loader, expected: anyerror) !void { try std.testing.expectEqual(expected, loader.failure.?); - try std.testing.expectEqual(0, loader.pending.len); + try std.testing.expectEqual(0, loader.pending.len + loader.queued.len + loader.programs.items.len); } }; -test "start publishes the model at once and wait retires it" { +test "start publishes the bulk at once and wait runs program groups one at a time" { const allocator = std.testing.allocator; const io = std.testing.io; var fixture: LoaderTestFixture = undefined; try fixture.init(allocator, io); defer fixture.deinit(); - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + // One program tensor per group. + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); defer loader.deinit(); - const Model = struct { value: Tensor, second: Tensor }; - const model: Model = .{ .value = fixture.value, .second = fixture.second }; + const Model = struct { value: Tensor, first: Tensor, second: Tensor }; + const model: Model = .{ + .value = fixture.value, + .first = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}), + .second = fixture.computed("second", fixture.second, LoaderTestFixture.Identity{}), + }; var buffers = try mem.bufferize(allocator, &model); defer mem.deinitBufferized(allocator, &buffers); try loader.start(&model, &buffers, &fixture.store, .none); errdefer loader.cancel(); + // The bulk is published; the groups wait for `wait`, their one program + // (the same function and shapes) compiling meanwhile. try std.testing.expectEqual(1, loader.pending.len); + try std.testing.expectEqual(2, loader.queued.len); + try std.testing.expectEqual(1, loader.programs.items.len); try loader.wait(); - try std.testing.expectEqual(0, loader.pending.len); + try std.testing.expectEqual(0, loader.pending.len + loader.queued.len + loader.programs.items.len); + // Admission retired the bulk and the first group before it published + // the second. + try std.testing.expectEqual(2, loader.stats.admission_retires); try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &buffers.first, &LoaderTestFixture.contents, .default); try testing.expectShards(allocator, io, &buffers.second, &LoaderTestFixture.second_contents, .default); - try std.testing.expectEqual(LoaderTestFixture.contents.len * 2, loader.stats.bytes_loaded); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); - // Idempotent: a second wait neither retires nor recounts. + // Idempotent: a second wait neither reruns nor recounts. try loader.wait(); - try std.testing.expectEqual(LoaderTestFixture.contents.len * 2, loader.stats.bytes_loaded); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); // The next load counts and times from its own first start. try std.testing.expectEqual(null, loader.stats.started_at); @@ -619,7 +1002,7 @@ test "one loader takes loads from different stores before one wait" { try testing.expectShards(allocator, io, &second, &LoaderTestFixture.second_contents, .default); } -test "cancel gives up on everything started" { +test "tensors of one program and equal args share one compile, and one group runs them all" { const allocator = std.testing.allocator; const io = std.testing.io; var fixture: LoaderTestFixture = undefined; @@ -628,35 +1011,143 @@ test "cancel gives up on everything started" { var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); defer loader.deinit(); - const Model = struct { value: Tensor, second: Tensor }; - const model: Model = .{ .value = fixture.value, .second = fixture.second }; + const Add = LoaderTestFixture.Add; + const Model = struct { a: Tensor, b: Tensor, c: Tensor }; + const model: Model = .{ + .a = fixture.computed("value", fixture.value, Add{ .value = 1 }), + .b = fixture.computed("second", fixture.second, Add{ .value = 1 }), + .c = fixture.computed("value", fixture.value, Add{ .value = 2 }), + }; + try std.testing.expect(fixture.store.bindingOf(model.a.id).? == .compute); + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + // Nothing for the loader to fill itself, and one group. + try std.testing.expectEqual(0, loader.pending.len); + try std.testing.expectEqual(1, loader.queued.len); + try std.testing.expectEqual(2, loader.programs.items.len); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.a, &.{ 2, 3, 4, 5 }, .default); + try testing.expectShards(allocator, io, &buffers.b, &.{ 6, 7, 8, 9 }, .default); + try testing.expectShards(allocator, io, &buffers.c, &.{ 3, 4, 5, 6 }, .default); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); +} + +test "a program frees its donated and non-donated inputs once" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); + const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); + defer allocator.free(path); + + const shape: Shape = .init(.{2}, .i32); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); + try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + + const Sum = struct { + donate: bool, + + pub fn forward(self: @This(), _: Shape, inputs: []const Tensor) Tensor { + const sum = inputs[0].add(inputs[1]); + return if (self.donate) sum.reuseBuffer(inputs[0]) else sum; + } + }; + for ([_]bool{ false, true }) |donate| { + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + const tensor = TensorStore.Source.bindProgram(&.{ + store.view().source("value", null).withPartitioning(.replicated, .{}), + store.view().source("weight", null).withPartitioning(.replicated, .{}), + }, shape, Sum{ .donate = donate }); + var output: Buffer = undefined; + try loader.start(&tensor, &output, &store, .none); + errdefer loader.cancel(); + try loader.programs.items[0].await(io, platform); + try std.testing.expectEqual(@as(usize, @intFromBool(donate)), loader.programs.items[0].exe.?.donated_input_indices.len); + // Retiring runs the program, then frees every input handle once, the + // donated one included. + try loader.wait(); + defer output.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + } +} + +test "cancel gives up on everything started without running its programs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Model = struct { value: Tensor, computed: Tensor }; + const model: Model = .{ .value = fixture.value, .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; var buffers = try mem.bufferize(allocator, &model); defer mem.deinitBufferized(allocator, &buffers); try loader.start(&model, &buffers, &fixture.store, .none); loader.cancel(); try LoaderTestFixture.expectFailed(&loader, error.Canceled); + try std.testing.expectEqual(@as(usize, 0), buffers.computed._shards.len); // The loader is failed: what it waits for and what it is given. try std.testing.expectError(error.Canceled, loader.wait()); try std.testing.expectError(error.Canceled, loader.start(&model, &buffers, &fixture.store, .none)); try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); } +test "cancel frees a published group's inputs without running its program" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Model = struct { computed: Tensor }; + const model: Model = .{ .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + // Publish the group as `wait` does, then give up before it retires. + try loader.publish(loader.queued.popFront().?); + try std.testing.expect(loader.programGroupPending()); + loader.cancel(); + try LoaderTestFixture.expectFailed(&loader, error.Canceled); + try std.testing.expect(!loader.programGroupPending()); + try std.testing.expectEqual(0, buffers.computed._shards.len); +} + test "a read failure fails the loader for every later call" { const allocator = std.testing.allocator; const io = std.testing.io; var fixture: LoaderTestFixture = undefined; try fixture.init(allocator, io); defer fixture.deinit(); - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + // The broken group first, retired when the next one is admitted. + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); defer loader.deinit(); const Model = struct { broken: Tensor, value: Tensor }; - const model: Model = .{ .broken = fixture.missing, .value = fixture.value }; + const model: Model = .{ + .broken = fixture.computed("missing", fixture.missing, LoaderTestFixture.Identity{}), + .value = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}), + }; var buffers = try mem.bufferize(allocator, &model); defer mem.deinitBufferized(allocator, &buffers); try loader.start(&model, &buffers, &fixture.store, .none); try std.testing.expectError(error.FileNotFound, loader.wait()); try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + try std.testing.expectEqual(@as(usize, 0), buffers.value._shards.len); try std.testing.expectError(error.FileNotFound, loader.start(&model, &buffers, &fixture.store, .none)); try std.testing.expectError(error.FileNotFound, loader.wait()); try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); @@ -672,14 +1163,18 @@ test "after a failed load every buffer is empty or whole, and deinitBufferized f defer loader.deinit(); // A tree with a slice and optionals, some of it in the file that loads, - // and a tensor in the file that does not exist. + // a tensor in the file that does not exist, and a program over it. const Layer = struct { weight: Tensor, bias: ?Tensor }; - const Model = struct { layers: []Layer, broken: Tensor }; + const Model = struct { layers: []Layer, broken: Tensor, computed: Tensor }; var layers = [_]Layer{ .{ .weight = fixture.value, .bias = fixture.second }, .{ .weight = fixture.second, .bias = null }, }; - const model: Model = .{ .layers = &layers, .broken = fixture.missing }; + const model: Model = .{ + .layers = &layers, + .broken = fixture.missing, + .computed = fixture.computed("missing", fixture.missing, LoaderTestFixture.Identity{}), + }; var buffers = try mem.bufferize(allocator, &model); // Frees whatever the failed load wrote, under the testing allocator. defer mem.deinitBufferized(allocator, &buffers); @@ -695,6 +1190,7 @@ test "after a failed load every buffer is empty or whole, and deinitBufferized f } }.call, .{ devices, &torn }); try std.testing.expectEqual(0, torn); + try std.testing.expectEqual(0, buffers.computed._shards.len); } test "init checks every option and returns InvalidOptions" { @@ -767,15 +1263,21 @@ test "without concurrency init sets up in place" { try testing.expectShards(allocator, io, &output, &LoaderTestFixture.contents, .default); } -test "a rejected binding fails the loader" { +test "a rejected binding or a program of the wrong output fails the loader" { const allocator = std.testing.allocator; const io = std.testing.io; var fixture: LoaderTestFixture = undefined; try fixture.init(allocator, io); defer fixture.deinit(); const platform = fixture.platform; + // Two rows of two, not the four bytes the tensor holds. + const Rows = struct { + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0].reshape(.{ 2, 2 }); + } + }; // A mesh of devices the platform does not have: the same tensors placed - // on it, in device and host memory. + // on it, in device and host memory, and a program's output. var foreign_physical = platform.replicated_mesh.physical.*; const foreign_devices = try allocator.dupe(Sharding.Device, foreign_physical.devices_in_canonical_order); defer allocator.free(foreign_devices); @@ -785,27 +1287,42 @@ test "a rejected binding fails the loader" { foreign_mesh.physical = &foreign_physical; const pinned = fixture.store.view().createReplicatedTensor("second", null); fixture.store.view().setMemory(pinned, .host_pinned); - var foreign = [_]Tensor{ fixture.value, pinned }; + var foreign = [_]Tensor{ fixture.value, pinned, fixture.value }; for (&foreign) |*tensor| tensor._shape._sharding.mesh = &foreign_mesh; + // The program places its output replicated, the tensor expects its + // bytes split over the devices: same dims, another placement. It needs + // devices that split four bytes. + var misplaced = fixture.value; + misplaced._shape = Shape.init(.{ .x = 4 }, .u8).withPartitioning(platform.meshes.get("model").?, .{ .x = .model }); + const splits = platform.devices.len > 1 and 4 % platform.devices.len == 0; - // No binding, an empty source, a mesh of other devices, a host kind the - // platform cannot take, a broadcast's strided runs and a program. - const Case = struct { tensor: Tensor, skip: bool = false }; + // No binding, an empty source or program input, a mesh of other + // devices, a host kind the platform cannot take, a broadcast's strided + // runs, and a program of the wrong shape or placement. + const Case = struct { tensor: Tensor, at_wait: bool = false, skip: bool = false }; const cases = [_]Case{ .{ .tensor = Tensor.fromShape(fixture.value.shape()) }, .{ .tensor = fixture.empty }, + .{ .tensor = fixture.computed("empty", fixture.empty, LoaderTestFixture.Identity{}) }, .{ .tensor = foreign[0] }, .{ .tensor = foreign[1] }, + .{ .tensor = fixture.computed("value", foreign[2], LoaderTestFixture.Identity{}) }, .{ .tensor = pinned, .skip = backend.supportsMemory(platform, .host_pinned) }, .{ .tensor = fixture.store.view().source("value", .{.x}).broad(Shape.init(.{ .y = 2, .x = 4 }, .u8)).withPartitioning(.replicated, .{}).bind() }, - .{ .tensor = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }, + .{ .tensor = fixture.computed("value", fixture.value, Rows{}), .at_wait = true }, + .{ .tensor = fixture.computed("value", misplaced, LoaderTestFixture.Identity{}), .at_wait = true, .skip = !splits }, }; for (cases) |case| { if (case.skip) continue; var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); defer loader.deinit(); var output: Buffer = undefined; - try std.testing.expectError(error.InvalidBinding, loader.start(&case.tensor, &output, &fixture.store, .none)); + if (case.at_wait) { + try loader.start(&case.tensor, &output, &fixture.store, .none); + try std.testing.expectError(error.InvalidBinding, loader.wait()); + } else { + try std.testing.expectError(error.InvalidBinding, loader.start(&case.tensor, &output, &fixture.store, .none)); + } try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); // `start` emptied the buffer before it failed. try std.testing.expectEqual(0, output._shards.len); @@ -862,3 +1379,42 @@ fn expectStructureMismatch(fixture: *LoaderTestFixture, comptime Model: type, bu }.call, .{ untouched, &written }); try std.testing.expectEqual(0, written); } + +test "a program takes its inputs from sources, in order" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const Source = TensorStore.Source; + const view = fixture.store.view(); + + // value's first half then second's last half, assembled by the loader, + // and second as it is: the program subtracts the second input from the + // first, which tells them apart. + const Sub = struct { + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0].sub(inputs[1]); + } + }; + const assembled = Source.concatenate(&.{ + view.source("value", .{.x}).slice(.x, .{ .end = 2 }), + view.source("second", .{.x}).slice(.x, .{ .start = 2 }), + }, .x); + const second = view.source("second", .{.x}); + const inputs = [_]Source.Placed{ assembled.withPartitioning(.replicated, .{}), second.withPartitioning(.replicated, .{}) }; + const tensor = Source.bindProgram(&inputs, second.shape(), Sub{}); + const compute = fixture.store.bindingOf(tensor.id).?.compute; + try std.testing.expectEqual(2, compute.inputs.len); + try std.testing.expect(compute.program.as(Sub) != null and compute.program.as(LoaderTestFixture.Identity) == null); + try std.testing.expect(compute.inputs[0].whole() == null); + try std.testing.expect(compute.inputs[1].whole() != null); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var output: Buffer = undefined; + try loader.start(&tensor, &output, &fixture.store, .none); + try loader.wait(); + defer output.deinit(); + // { 1, 2, 7, 8 } - { 5, 6, 7, 8 }, wrapping. + try testing.expectShards(allocator, io, &output, &.{ 252, 252, 0, 0 }, .default); +} diff --git a/zml/platform.zig b/zml/platform.zig index b2bd3a4e..6f9d114d 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_rocm = @import("platforms/rocm"); const stdx = @import("stdx"); pub const Target = @import("platforms").Platform; @@ -304,6 +305,16 @@ pub const Platform = struct { meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), replicated_mesh: *const zml.Mesh, io_impl: IoImpl, + /// Whether `releaseFreedDeviceMemory` asks the devices to give freed + /// memory back. `init` sets it for XLA's VMM allocator on CUDA and ROCm + /// (`CreateOptions.XlaGpu.Allocator.vmm`), which keeps freed memory + /// mapped, and counted in use, until it is reused or reclaimed. Null + /// otherwise: other allocators free at once, and oneAPI's VMM allocator + /// is not proven to follow the same rule. The flag lives in the + /// platform's arena, not in `Platform`: the first device that does not + /// answer as the VMM allocator does clears it for good, through a + /// `*const Platform`, without writing to the platform. + releases_freed_memory: ?*std.atomic.Value(bool) = null, pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; @@ -355,6 +366,16 @@ pub const Platform = struct { errdefer platform.arena.deinit(); errdefer if (comptime Target.cuda.isEnabled()) platform.state.deinit(); try platform.meshes.ensureTotalCapacity(arena, 8); + const keeps_freed_memory = switch (target) { + .cuda => options.cuda.allocator == .vmm, + .rocm => options.rocm.allocator == .vmm, + else => false, + }; + if (keeps_freed_memory) { + const releases = try arena.create(std.atomic.Value(bool)); + releases.* = .init(true); + platform.releases_freed_memory = releases; + } { const devices = try arena.alloc(Device, pjrt_devices.len); @@ -467,6 +488,77 @@ pub const Platform = struct { if (first_error.get()) |err| return err; } + /// Gives back to the devices memory that nothing uses but that stays + /// held: the freed buffers XLA's VMM allocator keeps, which + /// `Device.memoryStats` counts in `bytes_in_use`, and on ROCm what the + /// HIP runtime keeps. + /// + /// XLA's VMM allocator keeps a freed allocation mapped, and counted in + /// use, until a request of the same size reuses it, a request beyond + /// the device's limit reclaims it, or an executable with remapped + /// command buffers is destroyed. A request one byte beyond the limit + /// completes the deferred frees of the device's memory, once its + /// compute stream reached them, then fails with `ResourceExhausted`, as + /// expected. It leaves out the frees of collective memory, which stay + /// counted, and of pinned host memory, which `bytes_in_use` does not + /// count. Any other answer means the plugin does not follow that rule: + /// it is logged once, naming the device, and clears + /// `releases_freed_memory`, so later calls skip this part. This part is + /// a no-op for other allocators, which free at once, and for a device + /// that reports no limit. + /// + /// On ROCm, whatever the allocator, it first makes the HIP runtime give + /// back the device memory it keeps outside the allocator, chiefly the + /// scratch memory of kernels that spill registers, which `memoryStats` + /// never counts but the devices do + /// (`platforms_rocm.releaseRuntimeDeviceMemory`). That part reaches + /// every queue the process has on these devices, other platforms' + /// included, and makes running work wait: each queue holding scratch is + /// suspended and resumed, the call waits for its kernels using scratch, + /// and allocations on the device stall meanwhile. A call within a load + /// can overlap the load's other compiles and transfers, which then wait + /// too. A missing HIP runtime, or a device that does not refuse, is + /// logged once and ends that part for the process. + pub fn releaseFreedDeviceMemory(self: *const Platform) void { + if (comptime Target.rocm.isEnabled()) { + if (self.target == .rocm) _ = self.releaseRocmRuntimeMemory(); + } + const releases = self.releases_freed_memory orelse return; + for (self.devices) |*device| { + if (!releases.load(.monotonic)) return; + const limit = device.memoryStats().bytes_limit orelse continue; + // A limit that large is no limit: nothing goes beyond it. + const beyond_limit = std.math.cast(i64, limit + 1) orelse continue; + const dims: []const i64 = &.{beyond_limit}; + const answer: []const u8 = if (self.pjrt_client.createUninitializedBufferSilent(self.pjrt_api, .{ + .dims = dims, + .element_type = pjrtx.bufferTypeFromDtype(.u8), + .layout = self.defaultMemoryLayout(dims, .u8), + .dst = .{ .memory = device.memory(.default).?.pjrt_memory }, + })) |buffer| answer: { + buffer.deinit(self.pjrt_api); + break :answer "a buffer"; + } else |err| switch (err) { + error.ResourceExhausted => continue, + else => @errorName(err), + }; + if (releases.swap(false, .monotonic)) { + log.warn("Device {d} answered a request beyond its memory limit with {s}, not ResourceExhausted; freed device memory is no longer given back and may stay counted as in use", .{ device.id(), answer }); + } + return; + } + } + + /// `platforms_rocm.releaseRuntimeDeviceMemory` over every device: a + /// device's HIP ordinal is its local hardware id. + fn releaseRocmRuntimeMemory(self: *const Platform) bool { + var ordinals: [MAX_NUM_DEVICES]c_int = undefined; + for (self.devices, ordinals[0..self.devices.len]) |*device, *ordinal| { + ordinal.* = device.localHardwareId(); + } + return platforms_rocm.releaseRuntimeDeviceMemory(ordinals[0..self.devices.len]); + } + pub fn formatWithAttributes(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { const tee = "├─ "; const line = "│ "; @@ -1122,3 +1214,56 @@ test "platform device allocators can be warmed repeatedly" { try platform.warmupDeviceAllocators(std.testing.io); try platform.warmupDeviceAllocators(std.testing.io); } + +test "platform ROCm devices refuse a request for their whole memory, then still work" { + if (comptime !Target.rocm.isEnabled()) return error.SkipZigTest; + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + // Every device refuses a request for its whole memory, the request ROCr + // answers by reclaiming its queues' scratch and freeing its caches. The + // release itself is not visible from here: HIP counts ROCr's caches as + // free, and only the driver's counters in sysfs show it. + try std.testing.expect(platform.releaseRocmRuntimeMemory()); + + // The devices still take and give back data afterwards. + const values = [_]u32{ 1, 2, 3, 4, 5, 6, 7, 8 }; + var buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, .init(.{ .n = values.len }, .u32), std.mem.sliceAsBytes(&values)); + defer buffer.deinit(); + const read: zml.Slice = try buffer.toSliceAlloc(std.testing.allocator, std.testing.io); + defer read.free(std.testing.allocator); + try std.testing.expectEqualSlices(u32, &values, read.constItems(u32)); +} + +test "platform gives freed device memory back" { + const platform = zml.testing.env(); + // On GPU CI, CUDA and ROCm run this with XLA's VMM allocator. + const releases = platform.releases_freed_memory orelse return error.SkipZigTest; + const device = &platform.devices[0]; + // Without a limit the drain does nothing. + try std.testing.expect(device.memoryStats().bytes_limit != null); + + // An odd size, so that no request of the same size reuses it once freed. + const size = 64 * 1024 * 1024 + 4096; + const dims: []const i64 = &.{size}; + const buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, .{ + .dims = dims, + .element_type = pjrtx.bufferTypeFromDtype(.u8), + .layout = platform.defaultMemoryLayout(dims, .u8), + .dst = .{ .memory = device.memory(.default).?.pjrt_memory }, + }); + { + // Once its allocation is ready, destroying it frees it at once, + // not later on another thread. + errdefer buffer.deinit(platform.pjrt_api); + const ready = buffer.readyEvent(platform.pjrt_api); + defer ready.deinit(platform.pjrt_api); + try ready.await(platform.pjrt_api, std.testing.io); + } + buffer.deinit(platform.pjrt_api); + const in_use_freed = device.memoryStats().bytes_in_use; + + platform.releaseFreedDeviceMemory(); + try std.testing.expect(releases.load(.monotonic)); + try std.testing.expect(device.memoryStats().bytes_in_use + size <= in_use_freed); +} From b2309b7806facb1110e723344a9cae93dc81437e Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:47:15 +0100 Subject: [PATCH 061/103] zml/io: add the host Workspace with page arenas zml/io/host_memory.zig adds the host memory the direct backend will read into and transfer from: a Workspace of loader-owned arenas, mapped one at a time and then cut into blocks once. Nothing creates one yet outside its tests; io.zig's test block reaches it. CUDA and oneAPI arenas are pages registered with the PJRT client (DmaMap); CPU arenas are plain pageable pages. Arenas that span a transparent huge page are aligned to it and advised MADV_HUGEPAGE. A failed allocation or registration is logged with its cause and returns OutOfMemory; pages whose unregistration fails are leaked with a warning rather than reused. The other targets panic, since only the direct backend creates a workspace. allocate retains one arena and refuses a growth past a fixed 16 GiB mapped ceiling, a safety guard rather than a target. capacity counts the whole blocks the arenas hold and the ceiling still allows. carve maps the blocks missing below a first count as one arena, then those missing below the total as another, and cuts every arena into blocks: nothing maps after it, so nothing maps inside a load. The workspace owns the arenas and the block list. Co-Authored-By: Claude Opus 5.5 --- zml/BUILD.bazel | 1 + zml/io.zig | 1 + zml/io/host_memory.zig | 296 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 zml/io/host_memory.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 4c605a30..f8b647ca 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -91,6 +91,7 @@ zig_library( "io/TensorStore.zig", "io/backend.zig", "io/buffered_loader.zig", + "io/host_memory.zig", "io/loader.zig", "io/log.zig", "kernel.zig", diff --git a/zml/io.zig b/zml/io.zig index cca17b52..8cce359c 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -32,6 +32,7 @@ test { _ = @import("io/TensorStore.zig"); _ = @import("io/backend.zig"); _ = @import("io/buffered_loader.zig"); + _ = @import("io/host_memory.zig"); _ = @import("io/loader.zig"); _ = @import("io/log.zig"); } diff --git a/zml/io/host_memory.zig b/zml/io/host_memory.zig new file mode 100644 index 00000000..174291e5 --- /dev/null +++ b/zml/io/host_memory.zig @@ -0,0 +1,296 @@ +//! Loader-owned host arenas, cut into blocks once. The direct backend owns +//! its workspace for its whole life: it grows the set to what its load holds +//! and carves it, and its `destroy` releases every arena. Callers never +//! supply a workspace or a budget. +//! +//! The memory depends on the target: CUDA and oneAPI take page-backed arenas +//! registered with PJRT (DmaMap), and CPU plain pages, unregistered. +//! `Workspace.mapped_bytes_ceiling` (16 GiB) is a safety guard, not a +//! target. A failed allocation or registration, or a growth past the +//! ceiling, is logged with its cause and returns `OutOfMemory`. + +const std = @import("std"); +const Alignment = std.mem.Alignment; +const Allocator = std.mem.Allocator; +const builtin = @import("builtin"); + +const Platform = @import("../platform.zig").Platform; +const log = @import("log.zig").mem; + +/// One retained arena. +const Arena = struct { + bytes: []u8, +}; + +/// How arenas are mapped and unmapped; each strategy keeps only the state +/// it uses. +const ArenaKind = union(enum) { + /// Page-backed arenas, registered with the platform's PJRT client when + /// there is one (`dma_map` in the arena log) and left pageable for CPU + /// transfers when not (`pageable`), with huge-page alignment and advice. + /// Registration only avoids staging if the plugin recognizes the range as + /// pinned, including subranges. The oneAPI plugin must implement + /// IsHostMemoryPinned for SYCL imports: without it, one B70 staged + /// transfers at 11.01 GiB/s instead of 26.90 (eight DMA caps). + pages: struct { + platform: ?*const Platform, + }, + + const transparent_huge_page_size = 2 * 1024 * 1024; + + fn init(platform: *const Platform) ArenaKind { + return switch (platform.target) { + .cuda, .oneapi => .{ .pages = .{ .platform = platform } }, + .cpu => .{ .pages = .{ .platform = null } }, + .rocm, .furiosa, .tpu, .neuron, .metal => @panic("DMA is not accessible on this platform and shouldn't have been called."), + }; + } + + /// Maps one arena of `bytes`. + fn map(self: *ArenaKind, allocator: Allocator, io: std.Io, bytes: usize) Allocator.Error!Arena { + const started: std.Io.Timestamp = .now(io, .awake); + switch (self.*) { + .pages => |*pages| { + const alignment = pageAlignment(bytes); + const ptr = allocator.rawAlloc(bytes, alignment, @returnAddress()) orelse { + log.err("host memory allocation failed for {Bi:.2}: OutOfMemory", .{bytes}); + return error.OutOfMemory; + }; + const data = ptr[0..bytes]; + adviseHugePages(data); + if (pages.platform) |p| p.pjrt_client.dmaMap(p.pjrt_api, data) catch |err| { + log.err("DMA registration failed for {Bi:.2} of host memory: {s}", .{ bytes, @errorName(err) }); + allocator.rawFree(data, alignment, @returnAddress()); + return error.OutOfMemory; + }; + const finished: std.Io.Timestamp = .now(io, .awake); + log.debug("DMA arena kind={s} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ + if (pages.platform != null) "dma_map" else "pageable", + @intFromPtr(data.ptr), + bytes, + @as(f64, @floatFromInt(elapsedNanoseconds(started, finished))) / std.time.ns_per_ms, + }); + return .{ .bytes = data }; + }, + } + } + + fn unmap(self: *const ArenaKind, allocator: Allocator, arena: Arena) void { + switch (self.*) { + .pages => |pages| { + if (pages.platform) |platform| { + // Unregistering can fail after a device error. The pages may + // still be registered, so they are leaked, not reused. + platform.pjrt_client.dmaUnmap(platform.pjrt_api, arena.bytes) catch |err| { + log.warn("DMA unregistration failed for {Bi:.2} of host memory, which stays allocated: {s}", .{ arena.bytes.len, @errorName(err) }); + return; + }; + } + allocator.rawFree(arena.bytes, pageAlignment(arena.bytes.len), @returnAddress()); + }, + } + } + + /// Page alignment, raised to the huge-page size for arenas that span one. + fn pageAlignment(len: usize) Alignment { + const alignment: Alignment = comptime .fromByteUnits(std.heap.page_size_min); + if (comptime builtin.os.tag != .linux) return alignment; + if (len < transparent_huge_page_size) return alignment; + return alignment.max(.fromByteUnits(transparent_huge_page_size)); + } + + fn adviseHugePages(data: []u8) void { + if (comptime builtin.os.tag != .linux) return; + if (data.len < transparent_huge_page_size) return; + + const ptr: [*]align(std.heap.page_size_min) u8 = @alignCast(data.ptr); + std.posix.madvise(ptr, data.len, std.posix.MADV.HUGEPAGE) catch |err| { + log.warn("MADV_HUGEPAGE failed for DMA buffer at 0x{x} ({Bi:.2}): {s}", .{ + @intFromPtr(data.ptr), + data.len, + @errorName(err), + }); + }; + } +}; + +/// Host arenas owned by one direct loader. Arenas are mapped one at a time +/// (`allocate`); the loader then grows the set to what its load holds and +/// cuts it into blocks, once (`carve`). Deinitialize after all transfers +/// finish and before the platform. +pub const Workspace = struct { + /// Safety guard on the arenas' total host memory (pinned on the DMA + /// targets), not an allocation target: the load's pre-growth decides + /// what is actually mapped. Fixed, because no caller ever had a reason + /// to choose another value. + pub const mapped_bytes_ceiling: usize = 16 * 1024 * 1024 * 1024; + + allocator: Allocator, + io: std.Io, + kind: ArenaKind, + /// Every arena mapped, in mapping order, until `deinit`. + arenas: std.ArrayList(Arena) = .empty, + /// The arenas cut into blocks, set by `carve`: nothing maps after it. + blocks: ?[]Block = null, + + pub fn init( + allocator: Allocator, + io: std.Io, + platform: *const Platform, + ) Workspace { + return .{ + .allocator = allocator, + .io = io, + .kind = .init(platform), + }; + } + + pub fn deinit(self: *Workspace) void { + const io = self.io; + const mapped_bytes = self.mappedBytes(); + const started: std.Io.Timestamp = .now(io, .awake); + if (self.blocks) |blocks| self.allocator.free(blocks); + for (self.arenas.items) |arena| self.kind.unmap(self.allocator, arena); + self.arenas.deinit(self.allocator); + const elapsed_ns = elapsedNanoseconds(started, .now(io, .awake)); + log.debug("DMA load workspace teardown: mapped={Bi:.2}, elapsed_ms={d:.3}", .{ + mapped_bytes, + @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms, + }); + self.* = undefined; + } + + /// The bytes of every retained arena, tails included. + pub fn mappedBytes(self: *const Workspace) usize { + var mapped: usize = 0; + for (self.arenas.items) |arena| mapped += arena.bytes.len; + return mapped; + } + + /// Retains one new arena within the mapped-byte ceiling, before `carve`. + pub fn allocate(self: *Workspace, bytes: usize) Allocator.Error![]u8 { + std.debug.assert(self.blocks == null); + const mapped = self.mappedBytes(); + if (bytes > mapped_bytes_ceiling - mapped) { + log.err("host memory allocation exceeds mapped ceiling: requested={Bi:.2}, mapped={Bi:.2}, ceiling={Bi:.2}: OutOfMemory", .{ bytes, mapped, mapped_bytes_ceiling }); + return error.OutOfMemory; + } + // Reserved first, so a mapped arena is always retained. + try self.arenas.ensureUnusedCapacity(self.allocator, 1); + const arena = try self.kind.map(self.allocator, self.io, bytes); + self.arenas.appendAssumeCapacity(arena); + return arena.bytes; + } + + /// The blocks of `block_size` the workspace can hold: those of its + /// retained arenas, and those the mapped ceiling still allows. + pub fn capacity(self: *const Workspace, block_size: usize) usize { + return self.usableBlocks(block_size) + (mapped_bytes_ceiling - self.mappedBytes()) / block_size; + } + + /// Maps the blocks missing below `first_blocks` as one arena, then those + /// missing below `total_blocks` as another, and cuts every retained arena + /// into blocks of `block_size`, once. The set is then complete: nothing + /// maps inside a load (a slab mapped there cost 146 to 230 ms of + /// hipHostMalloc on MI300X), and `allocate` asserts it. Arena tails + /// smaller than one block stay mapped and unused. The workspace owns the + /// list and the memory. + pub fn carve(self: *Workspace, block_size: usize, first_blocks: usize, total_blocks: usize) Allocator.Error![]const Block { + std.debug.assert(self.blocks == null and block_size != 0); + for ([_]usize{ first_blocks, total_blocks }) |target_blocks| { + const missing_blocks = target_blocks -| self.usableBlocks(block_size); + if (missing_blocks != 0) _ = try self.allocate(missing_blocks * block_size); + } + const blocks = try self.allocator.alloc(Block, self.usableBlocks(block_size)); + var next: usize = 0; + for (self.arenas.items) |arena| { + for (0..arena.bytes.len / block_size) |j| { + blocks[next] = arena.bytes[j * block_size ..][0..block_size]; + next += 1; + } + } + std.debug.assert(next == blocks.len); + self.blocks = blocks; + return blocks; + } + + /// Counts complete blocks across retained arenas; block_size must be nonzero. + fn usableBlocks(self: *const Workspace, block_size: usize) usize { + var usable: usize = 0; + for (self.arenas.items) |arena| usable += arena.bytes.len / block_size; + return usable; + } + + /// Creates ordinary allocator-backed arenas for tests without a PJRT platform. + fn initForTesting(allocator: Allocator, io: std.Io) Workspace { + return .{ + .allocator = allocator, + .io = io, + .kind = .{ .pages = .{ .platform = null } }, + }; + } +}; + +pub const Block = []u8; + +fn elapsedNanoseconds(started: std.Io.Timestamp, finished: std.Io.Timestamp) u64 { + return @intCast(@max(started.durationTo(finished).nanoseconds, 0)); +} + +test "Workspace capacity counts whole blocks under the mapped ceiling, and carve refuses one more" { + const allocator = std.testing.allocator; + var workspace: Workspace = .initForTesting(allocator, std.testing.io); + defer workspace.deinit(); + const ceiling = Workspace.mapped_bytes_ceiling; + + try std.testing.expectEqual(ceiling / 64, workspace.capacity(64)); + _ = try workspace.allocate(150); + _ = try workspace.allocate(127); + // Whole blocks only: the arenas hold three, their tails none. + try std.testing.expectEqual(3 + (ceiling - 277) / 64, workspace.capacity(64)); + // Capacity is tight, as direct `create` sizes its carve from it: one + // block more crosses the ceiling (reaching it would map 16 GiB). + try std.testing.expectError(error.OutOfMemory, workspace.allocate(ceiling - 277 + 1)); + try std.testing.expectError(error.OutOfMemory, workspace.carve(64, workspace.capacity(64) + 1, 0)); + try std.testing.expectEqual(@as(usize, 277), workspace.mappedBytes()); +} + +test "Workspace carve maps each step's missing blocks as one arena and leaves tails out" { + const allocator = std.testing.allocator; + var workspace: Workspace = .initForTesting(allocator, std.testing.io); + defer workspace.deinit(); + const first = try workspace.allocate(150); + const second = try workspace.allocate(127); + + // Three blocks are mapped: two more make the first five, three more the + // eight. + const blocks = try workspace.carve(64, 5, 8); + try std.testing.expectEqual(@as(usize, 8), blocks.len); + try std.testing.expectEqual(@as(usize, 4), workspace.arenas.items.len); + try std.testing.expectEqual(@as(usize, 2 * 64), workspace.arenas.items[2].bytes.len); + try std.testing.expectEqual(@as(usize, 3 * 64), workspace.arenas.items[3].bytes.len); + for (blocks) |block| try std.testing.expectEqual(@as(usize, 64), block.len); + // Tails stay mapped without contributing a block. + try std.testing.expectEqual(first.ptr, blocks[0].ptr); + try std.testing.expectEqual(first.ptr + 64, blocks[1].ptr); + try std.testing.expectEqual(second.ptr, blocks[2].ptr); +} + +test "Workspace arena ownership cleans up allocation failures" { + const AllocationTest = struct { + fn run(allocator: Allocator) !void { + var workspace: Workspace = .initForTesting(allocator, std.testing.io); + defer workspace.deinit(); + + _ = try workspace.allocate(64); + _ = try workspace.allocate(128); + // The first step is already mapped; the second maps the last + // block. + const blocks = try workspace.carve(64, 3, 4); + try std.testing.expectEqual(@as(usize, 4), blocks.len); + try std.testing.expectEqual(@as(usize, 256), workspace.mappedBytes()); + try std.testing.expectEqual(@as(usize, 3), workspace.arenas.items.len); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, AllocationTest.run, .{}); +} From 0bc5b997ae9e942986229f177c8a8b7d4541d659 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:49:44 +0100 Subject: [PATCH 062/103] zml/io: add pjrt_host arenas for ROCm and Furiosa On ROCm and Furiosa, the Workspace's arenas are PJRT pinned_host buffers: an uninitialized u8 buffer in a device's host_pinned memory, retained through an external reference until the workspace unmaps it, whose host pointer the blocks use. Registering ordinary pages instead was far slower on ROCm: with eight MI300X visible, hipHostRegister took about 6.3 s per GiB against 0.6-0.7 s for hipHostMalloc. The pointer belongs to the PJRT buffer and is never DmaMapped, since unregistering allocation-owned memory breaks the eventual hipHostFree. Furiosa backs host_pinned with its SDK's dma-buf memory, which it recognizes by address. The workspace keeps one allocation path per host NUMA node its devices report (Device.numaNode), or a single one when none does, and maps each arena on the node with the fewest bytes so far: arenas differ in size, and balancing bytes rather than arenas or devices kept replicated loads on eight MI300X in their fast mode. Workspace.init can now fail with OutOfMemory. Transfers avoid staging only when the ROCm plugin recognizes these ranges as pinned (IsHostMemoryPinned); no CPU test reaches this arm. Co-Authored-By: Claude Opus 5.5 --- zml/io/host_memory.zig | 138 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 129 insertions(+), 9 deletions(-) diff --git a/zml/io/host_memory.zig b/zml/io/host_memory.zig index 174291e5..18061970 100644 --- a/zml/io/host_memory.zig +++ b/zml/io/host_memory.zig @@ -4,7 +4,9 @@ //! supply a workspace or a budget. //! //! The memory depends on the target: CUDA and oneAPI take page-backed arenas -//! registered with PJRT (DmaMap), and CPU plain pages, unregistered. +//! registered with PJRT (DmaMap), ROCm and Furiosa PJRT `pinned_host` +//! buffers, and CPU plain pages, unregistered. ROCm balances whole arenas +//! across its devices' host nodes. //! `Workspace.mapped_bytes_ceiling` (16 GiB) is a safety guard, not a //! target. A failed allocation or registration, or a growth past the //! ceiling, is logged with its cause and returns `OutOfMemory`. @@ -14,17 +16,40 @@ const Alignment = std.mem.Alignment; const Allocator = std.mem.Allocator; const builtin = @import("builtin"); +const pjrt = @import("pjrt"); + const Platform = @import("../platform.zig").Platform; const log = @import("log.zig").mem; -/// One retained arena. +/// One ROCm host-memory allocation path and its bytes allocated so far. +const HostNode = struct { + /// The plugin's `numa_node` device attribute (`Device.numaNode`); null + /// for the one path kept when no device reports it. + node: ?u32, + any_device_index: usize, + total_allocated_bytes: usize = 0, +}; + +/// One retained arena. The bytes of a `pinned_host` arena belong to its +/// PJRT buffer; page-backed arenas have none. const Arena = struct { bytes: []u8, + pjrt_buffer: ?*pjrt.Buffer = null, }; /// How arenas are mapped and unmapped; each strategy keeps only the state /// it uses. const ArenaKind = union(enum) { + /// With eight MI300X visible, hipHostRegister took ~6.3 s for 1 GiB + /// versus 0.6-0.7 s through hipHostMalloc. KFD/IOMMU registration maps + /// pages to every GPU; huge-page advice helped only ~9%, and touching + /// pages or changing HIP flags did not remove the cost. Standard PJRT + /// pinned_host buffers provide allocation-owned memory without a custom + /// allocator extension. + pjrt_host: struct { + platform: *const Platform, + host_nodes: std.ArrayList(HostNode), + }, /// Page-backed arenas, registered with the platform's PJRT client when /// there is one (`dma_map` in the arena log) and left pageable for CPU /// transfers when not (`pageable`), with huge-page alignment and advice. @@ -38,18 +63,103 @@ const ArenaKind = union(enum) { const transparent_huge_page_size = 2 * 1024 * 1024; - fn init(platform: *const Platform) ArenaKind { + fn init(allocator: Allocator, platform: *const Platform) Allocator.Error!ArenaKind { return switch (platform.target) { .cuda, .oneapi => .{ .pages = .{ .platform = platform } }, .cpu => .{ .pages = .{ .platform = null } }, - .rocm, .furiosa, .tpu, .neuron, .metal => @panic("DMA is not accessible on this platform and shouldn't have been called."), + // Furiosa backs `.host_pinned` with SDK pinned (dma-buf) memory + // from `/dev/dma_heap/system`, which the SDK recognizes by + // address and transfers without pinning pages on every request. + .rocm, .furiosa => { + // PJRT-pinned allocations choose placement through the device's + // host memory space. Retain one allocation path per reported + // node; the blocks stay unpartitioned. + var host_nodes: std.ArrayList(HostNode) = try .initCapacity(allocator, @max(platform.devices.len, 1)); + devices: for (platform.devices, 0..) |device, device_index| { + std.debug.assert(device.memory(.host_pinned) != null); + const node = device.numaNode() orelse continue; + for (host_nodes.items) |existing| { + if (existing.node == node) continue :devices; + } + host_nodes.appendAssumeCapacity(.{ .node = node, .any_device_index = device_index }); + } + if (host_nodes.items.len == 0) host_nodes.appendAssumeCapacity(.{ .node = null, .any_device_index = 0 }); + return .{ .pjrt_host = .{ .platform = platform, .host_nodes = host_nodes } }; + }, + .tpu, .neuron, .metal => @panic("DMA is not accessible on this platform and shouldn't have been called."), }; } + /// Call after every arena is unmapped. + fn deinit(self: *ArenaKind, allocator: Allocator) void { + switch (self.*) { + .pjrt_host => |*host| host.host_nodes.deinit(allocator), + .pages => {}, + } + } + /// Maps one arena of `bytes`. fn map(self: *ArenaKind, allocator: Allocator, io: std.Io, bytes: usize) Allocator.Error!Arena { const started: std.Io.Timestamp = .now(io, .awake); switch (self.*) { + .pjrt_host => |*host| { + // Balance bytes, not arena/device counts: arenas differ in size. + // Replicated Llama-3.1-8B on eight MI300X lost the fast mode when + // all arenas came from device zero (1.27-1.45 s). Device rotation + // left a 61/39 byte split and took 0.98-1.33 s; byte-balanced + // allocation recovered 0.84-1.05 s over six runs, with 1.52 GiB + // mapped versus 2.03 GiB for per-node pools. Host contention + // affected both arms. + var host_node = &host.host_nodes.items[0]; + for (host.host_nodes.items[1..]) |*candidate| { + if (candidate.total_allocated_bytes < host_node.total_allocated_bytes) host_node = candidate; + } + const device_index = host_node.any_device_index; + // PJRT is the sole owner of these hipHostMalloc-backed bytes. + // Never DmaMap or DmaUnmap the borrowed pointer: unregistering + // allocation-owned memory breaks the eventual hipHostFree. The + // buffer and its external reference both live until `unmap`, + // after all reads and DMA finish. + // The ROCm plugin must implement IsHostMemoryPinned: without + // it, one MI300X staged transfers at 6.5 GiB/s instead of 46.6. + const api = host.platform.pjrt_api; + const memory = host.platform.devices[device_index].memory(.host_pinned).?; + const buffer = host.platform.pjrt_client.createUninitializedBuffer(api, .{ + .dims = &.{@intCast(bytes)}, + .element_type = .u8, + .layout = .{ + .tiled = .{ + .minor_to_major = &.{0}, + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .dst = .{ .memory = memory.pjrt_memory }, + }) catch |err| { + log.err("pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); + return error.OutOfMemory; + }; + errdefer buffer.deinit(api); + std.debug.assert(buffer.isOnCpu(api)); + buffer.increaseExternalReferenceCount(api) catch |err| { + log.err("retaining pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); + return error.OutOfMemory; + }; + errdefer buffer.decreaseExternalReferenceCount(api) catch {}; + const ptr: [*]u8 = @ptrCast(buffer.opaqueDeviceMemoryDataPointer(api) catch |err| { + log.err("accessing pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); + return error.OutOfMemory; + }); + // Counted once mapped: the workspace retains every mapped arena. + host_node.total_allocated_bytes += bytes; + log.debug("DMA arena kind=pjrt_host device={d} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ + device_index, + @intFromPtr(ptr), + bytes, + @as(f64, @floatFromInt(elapsedNanoseconds(started, .now(io, .awake)))) / std.time.ns_per_ms, + }); + return .{ .bytes = ptr[0..bytes], .pjrt_buffer = buffer }; + }, .pages => |*pages| { const alignment = pageAlignment(bytes); const ptr = allocator.rawAlloc(bytes, alignment, @returnAddress()) orelse { @@ -77,6 +187,12 @@ const ArenaKind = union(enum) { fn unmap(self: *const ArenaKind, allocator: Allocator, arena: Arena) void { switch (self.*) { + .pjrt_host => |host| { + const api = host.platform.pjrt_api; + const buffer = arena.pjrt_buffer.?; + buffer.decreaseExternalReferenceCount(api) catch unreachable; + buffer.deinit(api); + }, .pages => |pages| { if (pages.platform) |platform| { // Unregistering can fail after a device error. The pages may @@ -137,11 +253,11 @@ pub const Workspace = struct { allocator: Allocator, io: std.Io, platform: *const Platform, - ) Workspace { + ) Allocator.Error!Workspace { return .{ .allocator = allocator, .io = io, - .kind = .init(platform), + .kind = try .init(allocator, platform), }; } @@ -152,6 +268,7 @@ pub const Workspace = struct { if (self.blocks) |blocks| self.allocator.free(blocks); for (self.arenas.items) |arena| self.kind.unmap(self.allocator, arena); self.arenas.deinit(self.allocator); + self.kind.deinit(self.allocator); const elapsed_ns = elapsedNanoseconds(started, .now(io, .awake)); log.debug("DMA load workspace teardown: mapped={Bi:.2}, elapsed_ms={d:.3}", .{ mapped_bytes, @@ -175,7 +292,8 @@ pub const Workspace = struct { log.err("host memory allocation exceeds mapped ceiling: requested={Bi:.2}, mapped={Bi:.2}, ceiling={Bi:.2}: OutOfMemory", .{ bytes, mapped, mapped_bytes_ceiling }); return error.OutOfMemory; } - // Reserved first, so a mapped arena is always retained. + // Reserved first, so a mapped arena is always retained: the ROCm + // balancer counts the bytes it maps. try self.arenas.ensureUnusedCapacity(self.allocator, 1); const arena = try self.kind.map(self.allocator, self.io, bytes); self.arenas.appendAssumeCapacity(arena); @@ -193,8 +311,10 @@ pub const Workspace = struct { /// into blocks of `block_size`, once. The set is then complete: nothing /// maps inside a load (a slab mapped there cost 146 to 230 ms of /// hipHostMalloc on MI300X), and `allocate` asserts it. Arena tails - /// smaller than one block stay mapped and unused. The workspace owns the - /// list and the memory. + /// smaller than one block stay mapped and unused. Blocks are not split + /// per NUMA node: strict per-node pools duplicated source reserves and + /// constrained reads to the smallest node; placement belongs to arena + /// allocation. The workspace owns the list and the memory. pub fn carve(self: *Workspace, block_size: usize, first_blocks: usize, total_blocks: usize) Allocator.Error![]const Block { std.debug.assert(self.blocks == null and block_size != 0); for ([_]usize{ first_blocks, total_blocks }) |target_blocks| { From 10e4e0a8a773ec57d5f425cabbc7cdca1dc4fd74 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:54:38 +0100 Subject: [PATCH 063/103] zml/io: add the direct backend's coalescing planner zml/io/direct_loader.zig starts the direct backend, which will read a checkpoint's bytes into pinned host blocks and copy them to the devices with PJRT host-to-device transfer managers, as a pure planner. It plans whole tensors into device memory with exact reads. Nothing calls it yet; io.zig's test block reaches it. The backend works per source file. Parts, each one source range of a destination, are sorted by file, offset and size, and each file becomes one plan whose jobs are claimed in file order. Touching or overlapping ranges merge into one run, never across a gap or a file, and each run is cut into the fewest jobs of at most one request, at the latest boundary where every earlier range has ended that still leaves room for the rest: no part is split where a whole cut fits. A job's transfers place its bytes on the devices through the destination's ShardGrid, one per run and DMA block, merged where the block, the writers and both offsets continue, so parts adjacent in the file and in the destination go out as one transfer. Every transfer stays inside one block. Coalescing this way took DeepSeek-V4-Flash from about 69,445 source reads to 9,524, and cutting at tensor-safe boundaries kept 69,572 DMA pieces where a rigid grid made 78,665. Co-Authored-By: Claude Opus 5.5 --- zml/BUILD.bazel | 1 + zml/io.zig | 1 + zml/io/direct_loader.zig | 505 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 507 insertions(+) create mode 100644 zml/io/direct_loader.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f8b647ca..4e29611d 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -91,6 +91,7 @@ zig_library( "io/TensorStore.zig", "io/backend.zig", "io/buffered_loader.zig", + "io/direct_loader.zig", "io/host_memory.zig", "io/loader.zig", "io/log.zig", diff --git a/zml/io.zig b/zml/io.zig index 8cce359c..3fce4fe9 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -32,6 +32,7 @@ test { _ = @import("io/TensorStore.zig"); _ = @import("io/backend.zig"); _ = @import("io/buffered_loader.zig"); + _ = @import("io/direct_loader.zig"); _ = @import("io/host_memory.zig"); _ = @import("io/loader.zig"); _ = @import("io/log.zig"); diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig new file mode 100644 index 00000000..720208c3 --- /dev/null +++ b/zml/io/direct_loader.zig @@ -0,0 +1,505 @@ +const std = @import("std"); + +const Buffer = @import("../buffer.zig").Buffer; +const Platform = @import("../platform.zig").Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const backend = @import("backend.zig"); +const ShardGrid = @import("ShardGrid.zig"); + +/// One output buffer and what places it. Its parts may come from several +/// files, so several plans can write it. +const Destination = struct { + /// The caller's shape, which the output buffer keeps. + shape: Shape, + /// Where `shape`, packed and placed (`Platform.placedShape`), lands on + /// its devices: what PJRT allocates and the planner cuts. + grid: ShardGrid, + output: *Buffer, + /// What fills it, in ascending, disjoint destination order + /// (`backend.LoadSpec`). + parts: []Part, +}; + +/// One source range of a destination: what the planner sorts by file offset +/// and cuts into jobs. A plain tensor is one part covering its whole +/// destination. +const Part = struct { + source: *const safetensors.Tensor, + source_slot: *SourceSlot, + destination: *Destination, + /// The spec part's; its dimensions are borrowed until the await + /// (`backend.LoadSpec`). + lattice: backend.Lattice, + + /// The part's bytes in its file, which `backend.checkParts` bounds. + fn fileRange(self: *const Part) struct { start: u64, end: u64 } { + const start = self.source.offset + self.lattice.src_offset; + return .{ .start = start, .end = start + self.lattice.width }; + } +}; + +/// One source file: every job of a plan reads the file of its slot. +const SourceSlot = struct { + uri: []const u8, +}; + +/// One file's jobs in claim order with their transfer records, allocated by +/// the planner. +const Plan = struct { + /// A claimable job: its slice of the plan's transfers and what its + /// worker does for them. + const Job = struct { + /// Where the job's read starts. + file_offset: u64, + transfer_start: usize, + transfer_len: usize, + kind: Kind, + + /// How the job's bytes reach their destinations. + const Kind = union(enum) { + /// Read into pinned blocks in file order, where its transfers + /// address it, then copied to the devices by DMA. + dma_read: Read, + }; + + /// `len` bytes from the job's `file_offset`, of which the first + /// `minimum_len` hold tensor data and must exist. Reads are exact: + /// the two are equal. + const Read = struct { + len: usize, + minimum_len: usize, + }; + + /// The blocks the job reads into. + fn blockLen(self: Job, block_size: usize) usize { + return switch (self.kind) { + .dma_read => |read| Planner.blockCount(read.len, block_size), + }; + } + }; + + /// `len` bytes at `landing` of the job's blocks taken end to end, for + /// every device of `writer_mask` at `destination_offset` of its shard. + /// Every transfer stays inside one block. + const Transfer = struct { + destination: *Destination, + landing: usize, + writer_mask: u64, + destination_offset: usize, + len: usize, + }; + + /// The file every job of the plan reads. + source_slot: *SourceSlot, + jobs: []Job, + transfers: []Transfer, + /// The tensor bytes the jobs read. + source_bytes: u64, + + fn transfersOf(self: *const Plan, job: Job) []const Transfer { + return self.transfers[job.transfer_start..][0..job.transfer_len]; + } +}; + +/// Builds immutable per-file source jobs and transfer records. Their claim +/// order (file order) is decided here once. +/// For DeepSeek-V4-Flash (148.65 GiB, 69,187 tensors), tensor-local reads made +/// about 69,445 source calls despite a 16 MiB request limit. Coalescing reduced +/// that to 9,524. A rigid grid still produced 78,665 DMA pieces; tensor-safe +/// cuts kept the same read count with 69,572 pieces. Per-tensor device buffers +/// remain intentional: reducing submissions below roughly tensors x destinations would require +/// packed device allocations or device-side scattering, not another read cut. +const Planner = struct { + /// What every plan of a loader is cut by. + const Config = struct { + block_size: usize, + /// The longest read, a whole number of blocks + /// (`backend.requestSize`). + request_size: usize, + }; + + /// Plans one file: `order` indexes its `parts` by offset and size + /// (`sortedPartOrder`), and `JobCut.cutRuns` cuts them. The plan goes in + /// `arena` (`JobCut.finish`), the cut in `allocator`. + fn preparePlan( + allocator: std.mem.Allocator, + arena: std.mem.Allocator, + parts: []Part, + order: []const usize, + config: Config, + ) !*Plan { + const source_slot = parts[order[0]].source_slot; + for (order) |part_index| std.debug.assert(std.mem.eql(u8, parts[part_index].source.file_uri, parts[order[0]].source.file_uri)); + var cut: JobCut = .{ + .allocator = allocator, + .parts = parts, + .source_slot = source_slot, + .config = config, + }; + defer cut.deinit(); + // Jobs are claimed in planning order, which is file order. A + // byte-fair order across destination devices was measured against + // it on the two hosts where the DMA pump is not the ceiling and made + // no difference. + try cut.cutRuns(order); + return cut.finish(arena); + } + + /// The jobs and transfers of one plan as `preparePlan` cuts them. + const JobCut = struct { + /// One part's bytes in a read job: the logical range `[start, end)` + /// of its destination, from `file_offset` of the file on. + const Segment = struct { destination: *Destination, start: u64, end: u64, file_offset: u64 }; + + allocator: std.mem.Allocator, + parts: []Part, + /// The file every job of the plan reads. + source_slot: *SourceSlot, + config: Config, + jobs: std.ArrayList(Plan.Job) = .empty, + transfers: std.ArrayList(Plan.Transfer) = .empty, + safe_boundaries: std.ArrayList(u64) = .empty, + source_bytes: u64 = 0, + + /// The plan of the jobs cut, copied into `arena`. + fn finish(self: *JobCut, arena: std.mem.Allocator) !*Plan { + const plan = try arena.create(Plan); + plan.* = .{ + .source_slot = self.source_slot, + .jobs = try arena.dupe(Plan.Job, self.jobs.items), + .transfers = try arena.dupe(Plan.Transfer, self.transfers.items), + .source_bytes = self.source_bytes, + }; + return plan; + } + + fn deinit(self: *JobCut) void { + const allocator = self.allocator; + self.safe_boundaries.deinit(allocator); + self.transfers.deinit(allocator); + self.jobs.deinit(allocator); + } + + /// Cuts the runs of `order` (part indices in file order) into jobs: + /// each run of touching or overlapping ranges into the minimum number + /// of jobs of at most a request, cut at part-safe boundaries where + /// that number allows. No job depends on another, since every DMA + /// piece is submitted as soon as its block is read. + fn cutRuns(self: *JobCut, order: []const usize) !void { + const allocator = self.allocator; + const parts = self.parts; + const maximum_job_len = self.config.request_size; + var run_cursor: usize = 0; + while (run_cursor < order.len) { + self.safe_boundaries.clearRetainingCapacity(); + const first_range = parts[order[run_cursor]].fileRange(); + std.debug.assert(first_range.end > first_range.start); // backend.LoadSpec: parts are non-empty + const first_offset = first_range.start; + var run_end = first_range.end; + var run_part_end = run_cursor + 1; + while (run_part_end < order.len) : (run_part_end += 1) { + const candidate = parts[order[run_part_end]].fileRange(); + if (candidate.start > run_end) break; + // A touching range starts where every preceding range has + // ended. Unlike an arbitrary part end, this is safe even + // when the batch contains overlapping or duplicate ranges. + if (candidate.start == run_end and + (self.safe_boundaries.items.len == 0 or + self.safe_boundaries.items[self.safe_boundaries.items.len - 1] != candidate.start)) + { + try self.safe_boundaries.append(allocator, candidate.start); + } + run_end = @max(run_end, candidate.end); + } + + var job_start = first_offset; + var candidate_start = run_cursor; + var boundary_cursor: usize = 0; + const run_len = run_end - first_offset; + const maximum_job_len_u64: u64 = @intCast(maximum_job_len); + var jobs_remaining: usize = @intCast(run_len / maximum_job_len_u64 + + @intFromBool(run_len % maximum_job_len_u64 != 0)); + while (jobs_remaining != 0) { + const safe_boundaries = self.safe_boundaries.items; + const hard_end = @min( + run_end, + std.math.add(u64, job_start, maximum_job_len) catch run_end, + ); + const job_end = if (jobs_remaining == 1) + run_end + else boundary: { + // Less than the run's length, which is a u64. + const remaining_capacity = @as(u64, @intCast(jobs_remaining - 1)) * maximum_job_len_u64; + const minimum_end = @max(job_start + 1, run_end - remaining_capacity); + const maximum_end = @min( + hard_end, + run_end - @as(u64, @intCast(jobs_remaining - 1)), + ); + while (boundary_cursor < safe_boundaries.len and + safe_boundaries[boundary_cursor] <= job_start) + { + boundary_cursor += 1; + } + var scan = boundary_cursor; + var selected: ?u64 = null; + while (scan < safe_boundaries.len and + safe_boundaries[scan] <= maximum_end) : (scan += 1) + { + if (safe_boundaries[scan] >= minimum_end) + selected = safe_boundaries[scan]; + } + boundary_cursor = scan; + break :boundary selected orelse maximum_end; + }; + const job_len: usize = @intCast(job_end - job_start); + const transfer_start = self.transfers.items.len; + while (candidate_start < run_part_end) { + const candidate = parts[order[candidate_start]].fileRange(); + if (candidate.end > job_start) break; + candidate_start += 1; + } + for (order[candidate_start..run_part_end]) |part_index| { + const part = &parts[part_index]; + const range = part.fileRange(); + if (range.start >= job_end) break; + const intersection_start = @max(job_start, range.start); + const intersection_end = @min(job_end, range.end); + if (intersection_start >= intersection_end) continue; + const logical_start = part.lattice.dst_offset + (intersection_start - range.start); + try self.appendRunTransfers(transfer_start, .{ + .destination = part.destination, + .start = logical_start, + .end = logical_start + (intersection_end - intersection_start), + .file_offset = intersection_start, + }, job_start); + } + std.debug.assert(self.transfers.items.len != transfer_start); + try self.jobs.append(allocator, .{ + .file_offset = job_start, + .transfer_start = transfer_start, + .transfer_len = self.transfers.items.len - transfer_start, + .kind = .{ .dma_read = .{ .len = job_len, .minimum_len = job_len } }, + }); + self.source_bytes +|= @intCast(job_len); + job_start = job_end; + jobs_remaining -= 1; + } + std.debug.assert(job_start == run_end); + run_cursor = run_part_end; + } + } + + /// Appends `segment`'s transfers with its bytes where the read holds + /// them, one per run (`ShardGrid.run`) and block, merged with the + /// previous transfer from `transfer_start` on where block, writers + /// and both offsets continue it: parts adjacent in the file and in + /// the destination go out as one transfer. + fn appendRunTransfers(self: *JobCut, transfer_start: usize, segment: Segment, read_start: u64) !void { + const destination = segment.destination; + const grid = &destination.grid; + var at = segment.start; + while (at < segment.end) { + const stop = grid.runEnd(at, segment.end); + const box, const box_offset = grid.locate(at); + try self.appendBlocked(transfer_start, .{ + .destination = destination, + .landing = @intCast(segment.file_offset + (at - segment.start) - read_start), + .writer_mask = grid.writers[box], + .destination_offset = @intCast(box_offset), + .len = @intCast(stop - at), + }); + at = stop; + } + } + + /// Appends `transfer`, cut at block boundaries, merged with the + /// previous transfer from `transfer_start` on where block, writers + /// and both offsets continue it. + fn appendBlocked(self: *JobCut, transfer_start: usize, transfer: Plan.Transfer) !void { + const block_size = self.config.block_size; + var cursor: usize = 0; + while (cursor < transfer.len) { + const at = transfer.landing + cursor; + const in_block = at % block_size; + const take = @min(transfer.len - cursor, block_size - in_block); + const destination_offset = transfer.destination_offset + cursor; + cursor += take; + // A transfer that ends where this one starts is in its + // block unless a block starts here. + if (self.transfers.items.len > transfer_start and in_block != 0) { + const previous = &self.transfers.items[self.transfers.items.len - 1]; + if (previous.destination == transfer.destination and + previous.writer_mask == transfer.writer_mask and + previous.landing + previous.len == at and + previous.destination_offset + previous.len == destination_offset) + { + previous.len += take; + continue; + } + } + try self.transfers.append(self.allocator, .{ + .destination = transfer.destination, + .landing = at, + .writer_mask = transfer.writer_mask, + .destination_offset = destination_offset, + .len = take, + }); + } + } + }; + fn blockCount(len: usize, block_size: usize) usize { + return len / block_size + @intFromBool(len % block_size != 0); + } + + /// Part indices sorted by file URI, offset, size and index: the + /// planner's input, taken one file group at a time (`fileGroupEnd`), in + /// file order. + fn sortedPartOrder(allocator: std.mem.Allocator, parts: []const Part) ![]usize { + const order = try allocator.alloc(usize, parts.len); + for (order, 0..) |*index, i| index.* = i; + const SortContext = struct { + parts: []const Part, + + fn lessThan(ctx: @This(), lhs: usize, rhs: usize) bool { + const left = ctx.parts[lhs]; + const right = ctx.parts[rhs]; + const uri_order = std.mem.order(u8, left.source.file_uri, right.source.file_uri); + if (uri_order != .eq) return uri_order == .lt; + const left_offset = left.fileRange().start; + const right_offset = right.fileRange().start; + if (left_offset != right_offset) return left_offset < right_offset; + if (left.lattice.width != right.lattice.width) return left.lattice.width < right.lattice.width; + return lhs < rhs; + } + }; + std.mem.sort(usize, order, SortContext{ .parts = parts }, SortContext.lessThan); + return order; + } + + /// The end of the file group that begins at `order[start]`. + fn fileGroupEnd(parts: []const Part, order: []const usize, start: usize) usize { + const uri = parts[order[start]].source.file_uri; + var end = start + 1; + while (end < order.len and std.mem.eql(u8, uri, parts[order[end]].source.file_uri)) : (end += 1) {} + return end; + } +}; + +// Planner tests. + +/// `Planner.preparePlan`, kept in `arena`; the testing allocator checks the +/// cut for leaks. +fn prepareTestPlan(arena: std.mem.Allocator, parts: []Part, order: []const usize, config: Planner.Config) !*Plan { + return Planner.preparePlan(std.testing.allocator, arena, parts, order, config); +} + +/// One plain destination per source, as whole-tensor specs make them: for +/// planner tests. +fn initPlainParts( + platform: *const Platform, + sources: []safetensors.Tensor, + slots: []const *SourceSlot, + outputs: []Buffer, + destinations: []Destination, + parts: []Part, +) void { + for (sources, slots, outputs, destinations, parts) |*source, slot, *output, *destination, *part| { + part.* = .{ + .source = source, + .source_slot = slot, + .destination = destination, + .lattice = .{ .width = source.byteSize() }, + }; + destination.* = .{ + .shape = source.shape, + .grid = .init(platform, source.shape), + .output = output, + .parts = @as(*[1]Part, part), + }; + } +} + +test "source planner coalesces exact adjacent and overlapping tensor ranges per file" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "a", .name = "a0", .shape = .init(.{4}, .u8), .offset = 10 }, + .{ .file_uri = "a", .name = "a1", .shape = .init(.{4}, .u8), .offset = 14 }, + .{ .file_uri = "a", .name = "a0-copy", .shape = .init(.{4}, .u8), .offset = 10 }, + .{ .file_uri = "a", .name = "a-gap", .shape = .init(.{4}, .u8), .offset = 20 }, + .{ .file_uri = "b", .name = "b0", .shape = .init(.{12}, .u8), .offset = 3 }, + }; + var slots = [_]SourceSlot{ + .{ .uri = "a" }, + .{ .uri = "b" }, + }; + var outputs: [sources.len]Buffer = undefined; + var destinations: [sources.len]Destination = undefined; + var parts: [sources.len]Part = undefined; + initPlainParts(platform, &sources, &.{ &slots[0], &slots[0], &slots[0], &slots[0], &slots[1] }, &outputs, &destinations, &parts); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const order = try Planner.sortedPartOrder(allocator, &parts); + defer allocator.free(order); + var plans: [2]*Plan = undefined; + var file_start: usize = 0; + for (&plans) |*plan| { + const file_end = Planner.fileGroupEnd(&parts, order, file_start); + plan.* = try prepareTestPlan(arena.allocator(), &parts, order[file_start..file_end], .{ + .block_size = 4, + .request_size = 8, + }); + file_start = file_end; + } + try std.testing.expectEqual(order.len, file_start); + + // One plan per file. a:[10,18) merges adjacency and the duplicate, + // a:[20,24) remains exact, and b:[3,15) is split at the request-size + // boundary. + try std.testing.expect(plans[0].source_slot == &slots[0]); + try std.testing.expect(plans[1].source_slot == &slots[1]); + try std.testing.expectEqual(@as(usize, 2), plans[0].jobs.len); + try std.testing.expectEqual(@as(usize, 4), plans[0].transfers.len); + try std.testing.expectEqual(@as(u64, 10), plans[0].jobs[0].file_offset); + try std.testing.expectEqual(@as(usize, 8), plans[0].jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 3), plans[0].jobs[0].transfer_len); + try std.testing.expectEqual(@as(u64, 20), plans[0].jobs[1].file_offset); + try std.testing.expectEqual(@as(usize, 2), plans[1].jobs.len); + try std.testing.expectEqual(@as(usize, 3), plans[1].transfers.len); + try std.testing.expectEqual(@as(u64, 3), plans[1].jobs[0].file_offset); + try std.testing.expectEqual(@as(usize, 8), plans[1].jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 4), plans[1].jobs[1].kind.dma_read.len); + try std.testing.expectEqual(@as(u64, 12), plans[0].source_bytes); + try std.testing.expectEqual(@as(u64, 12), plans[1].source_bytes); + // Jobs span their 4-byte blocks. + try std.testing.expectEqual(@as(usize, 2), plans[0].jobs[0].blockLen(4)); + try std.testing.expectEqual(@as(usize, 1), plans[0].jobs[1].blockLen(4)); + try std.testing.expectEqual(@as(usize, 2), plans[1].jobs[0].blockLen(4)); + try std.testing.expectEqual(@as(usize, 1), plans[1].jobs[1].blockLen(4)); + + var aligned_sources = [_]safetensors.Tensor{ + .{ .file_uri = "aligned", .name = "aligned0", .shape = .init(.{7}, .u8), .offset = 0 }, + .{ .file_uri = "aligned", .name = "aligned1", .shape = .init(.{8}, .u8), .offset = 7 }, + .{ .file_uri = "aligned", .name = "aligned2", .shape = .init(.{5}, .u8), .offset = 15 }, + }; + var aligned_slot: SourceSlot = .{ .uri = "aligned" }; + var aligned_outputs: [aligned_sources.len]Buffer = undefined; + var aligned_destinations: [aligned_sources.len]Destination = undefined; + var aligned_items: [aligned_sources.len]Part = undefined; + initPlainParts(platform, &aligned_sources, &.{ &aligned_slot, &aligned_slot, &aligned_slot }, &aligned_outputs, &aligned_destinations, &aligned_items); + const aligned_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ + .block_size = 4, + .request_size = 8, + }); + try std.testing.expectEqual(@as(usize, 3), aligned_plan.jobs.len); + try std.testing.expectEqual(@as(usize, 6), aligned_plan.transfers.len); + try std.testing.expectEqual(@as(usize, 7), aligned_plan.jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 8), aligned_plan.jobs[1].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 5), aligned_plan.jobs[2].kind.dma_read.len); +} From 04cb4adec9bc17df70e69da6112b2a23161e3418 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:56:31 +0100 Subject: [PATCH 064/103] zml/io: add LazyOnce LazyOnce(T) is a value the first task to touch it initializes, with the initializer that task passes. Concurrent callers wait for it, and a failed initialization keeps its error code and returns the same error to every later caller. The direct backend will open each source file and create each destination's device buffers through it, from whichever worker reaches them first. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 67 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 720208c3..48debb23 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -386,6 +386,57 @@ const Planner = struct { } }; +/// A value initialized at most once by whichever task touches it first, +/// with the initializer that task passes. Concurrent callers wait on the +/// event; a failed initialization keeps its error code and re-materializes +/// the same error for every later caller. +fn LazyOnce(comptime T: type) type { + return struct { + const Self = @This(); + const Status = enum(u8) { + uninitialized, + initializing, + ready, + failed, + }; + + value: T = undefined, + status: std.atomic.Value(Status) = .init(.uninitialized), + error_code: std.atomic.Value(u16) = .init(0), + initialized: std.Io.Event = .unset, + + /// The error set of `initFn`, which returns `T`. + fn InitError(comptime initFn: anytype) type { + return @typeInfo(@typeInfo(@TypeOf(initFn)).@"fn".return_type.?).error_union.error_set; + } + + fn ensure(self: *Self, io: std.Io, comptime initFn: anytype, args: std.meta.ArgsTuple(@TypeOf(initFn))) InitError(initFn)!*T { + while (true) switch (self.status.load(.acquire)) { + .uninitialized => { + if (self.status.cmpxchgStrong(.uninitialized, .initializing, .acq_rel, .acquire) != null) continue; + self.value = @call(.auto, initFn, args) catch |err| { + self.error_code.store(@intFromError(err), .release); + self.status.store(.failed, .release); + self.initialized.set(io); + return err; + }; + self.status.store(.ready, .release); + self.initialized.set(io); + return &self.value; + }, + .initializing => self.initialized.waitUncancelable(io), + .ready => return &self.value, + .failed => return @errorCast(@errorFromInt(self.error_code.load(.acquire))), + }; + } + + /// The value when initialization has completed successfully. + fn readyValue(self: *Self) ?*T { + return if (self.status.load(.acquire) == .ready) &self.value else null; + } + }; +} + // Planner tests. /// `Planner.preparePlan`, kept in `arena`; the testing allocator checks the @@ -503,3 +554,19 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per try std.testing.expectEqual(@as(usize, 8), aligned_plan.jobs[1].kind.dma_read.len); try std.testing.expectEqual(@as(usize, 5), aligned_plan.jobs[2].kind.dma_read.len); } + +// LazyOnce tests. + +test "lazy initialization preserves its finite error across repeated callers" { + const Context = struct { + fn init(calls: *usize) error{FileNotFound}!u32 { + calls.* += 1; + return error.FileNotFound; + } + }; + var once: LazyOnce(u32) = .{}; + var calls: usize = 0; + try std.testing.expectError(error.FileNotFound, once.ensure(std.testing.io, Context.init, .{&calls})); + try std.testing.expectError(error.FileNotFound, once.ensure(std.testing.io, Context.init, .{&calls})); + try std.testing.expectEqual(1, calls); +} From 09c3bc9339f65446c43538959f909c8055e75f5f Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:58:20 +0100 Subject: [PATCH 065/103] zml/io: add readFilePositionalAllV readFilePositionalAllV reads a file range into a list of buffers in order, consuming the buffers as it goes. The io takes as many buffers per call as it can, and a short read resumes where it stopped. Reading stops once a minimum is in, so bytes a caller added past the data to align a read may stay unread where the file ends, and a read that ends before the minimum fails with UnexpectedEndOfFile. Every source read of the direct backend will go through it. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 66 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 48debb23..89131b76 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -386,6 +386,36 @@ const Planner = struct { } }; +/// Reads `[file_offset, file_offset + minimum)`, and possibly more, into +/// `buffers` in order, consuming them as it reads; bytes past `minimum` may +/// stay unread. The io takes as many buffers per call as it can (VFS's +/// direct read `IOV_MAX`, std's and zio's 8, a remote read all); a short +/// read resumes where it stopped. Reading stops once `minimum` bytes are +/// in, so the file may end only past `minimum`: the bytes after it are +/// padding a caller added to align the read, and the read that would fetch +/// them can start unaligned, which a file read directly refuses. Returns +/// the bytes read. +fn readFilePositionalAllV( + io: std.Io, + file: std.Io.File, + buffers: [][]u8, + file_offset: u64, + minimum: u64, +) !u64 { + var index: usize = 0; + var completed: u64 = 0; + while (completed < minimum) { + std.debug.assert(index < buffers.len); + const bytes_read = try file.readPositional(io, buffers[index..], file_offset + completed); + if (bytes_read == 0) return error.UnexpectedEndOfFile; + completed += bytes_read; + var rest = bytes_read; + while (index < buffers.len and rest >= buffers[index].len) : (index += 1) rest -= buffers[index].len; + if (rest != 0) buffers[index] = buffers[index][rest..]; + } + return completed; +} + /// A value initialized at most once by whichever task touches it first, /// with the initializer that task passes. Concurrent callers wait on the /// event; a failed initialization keeps its error code and re-materializes @@ -555,6 +585,42 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per try std.testing.expectEqual(@as(usize, 5), aligned_plan.jobs[2].kind.dma_read.len); } +// Source read tests. + +test "readFilePositionalAllV resumes short reads and stops at minimum" { + const OneByteIo = struct { + const file_end = 11; + + /// Reads one byte per call, the low byte of its offset, from a file + /// that ends at `file_end`, and counts the calls in `userdata`. + fn fileReadPositional(userdata: ?*anyopaque, _: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const calls: *usize = @ptrCast(@alignCast(userdata.?)); + calls.* += 1; + if (offset >= file_end) return 0; + for (data) |buffer| { + if (buffer.len == 0) continue; + buffer[0] = @truncate(offset); + return 1; + } + return 0; + } + }; + var vtable = std.testing.io.vtable.*; + vtable.fileReadPositional = OneByteIo.fileReadPositional; + var calls: usize = 0; + const io: std.Io = .{ .userdata = &calls, .vtable = &vtable }; + const file: std.Io.File = .{ .handle = 0, .flags = .{ .nonblocking = false } }; + + var output: [5]u8 = @splat(0xaa); + var buffers = [_][]u8{ output[0..1], output[1..1], output[1..3], output[3..5] }; + try std.testing.expectEqual(@as(u64, 4), try readFilePositionalAllV(io, file, &buffers, 7, 4)); + try std.testing.expectEqual(@as(usize, 4), calls); + try std.testing.expectEqualSlices(u8, &.{ 7, 8, 9, 10, 0xaa }, &output); + + var past_end = [_][]u8{&output}; + try std.testing.expectError(error.UnexpectedEndOfFile, readFilePositionalAllV(io, file, &past_end, 7, output.len)); +} + // LazyOnce tests. test "lazy initialization preserves its finite error across repeated callers" { From 9d72018c5be72533c5d9f3f2b135564cf5ec06e4 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 12:59:45 +0100 Subject: [PATCH 066/103] zml/io: add Credits Credits are the direct backend's lifecycle credits: each is a fixed slot of pinned blocks, one source request's worth. A worker will take a credit before it claims a job and its request gives it back after its last DMA callback, so only the holder reads into and transfers from a slot, and the credits are the one bound on the pinned memory in use. acquire waits for a free slot and release wakes one waiter, not all of them. They also count the most slots held at once. There can be more credits than reads in flight, so transfers still holding their blocks keep the DMA engines fed: on a GB300 loading DeepSeek-V4, one credit beyond the read width moved 24.3 GiB/s against 43.8 GiB/s with eight requests queued. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 89131b76..7e53a648 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -6,6 +6,7 @@ const safetensors = @import("../safetensors.zig"); const Shape = @import("../shape.zig").Shape; const backend = @import("backend.zig"); const ShardGrid = @import("ShardGrid.zig"); +const host_memory = @import("host_memory.zig"); /// One output buffer and what places it. Its parts may come from several /// files, so several plans can write it. @@ -416,6 +417,80 @@ fn readFilePositionalAllV( return completed; } +/// The lifecycle credits: one per request the pre-grown pinned set holds, +/// each a fixed slot of `blocks_per_slot` blocks. A worker takes a credit +/// before claiming a job, and the request gives it back with its last +/// reference, after its last DMA callback. Only the holder reads into and +/// transfers from its slot's blocks, so the credits are the one bound on +/// pinned memory in use. A block whose pieces are done stays with its +/// request until the request ends; nobody else could use it anyway, since +/// every credit owns a whole slot. +const Credits = struct { + /// Every slot's blocks, slot `s` at `blocks[s * blocks_per_slot ..]`; + /// a tail shorter than a slot is left unused. + blocks: []const host_memory.Block, + blocks_per_slot: usize, + /// Free slots, a stack of indices; owned by `mutex`. + free: []u32, + free_len: usize, + /// The most slots held at once. + high_water: usize = 0, + mutex: std.Io.Mutex = .init, + /// Waiters for a slot; one release wakes one of them. + condition: std.Io.Condition = .init, + + fn init(allocator: std.mem.Allocator, blocks: []const host_memory.Block, blocks_per_slot: usize) std.mem.Allocator.Error!Credits { + std.debug.assert(blocks_per_slot != 0); + const free = try allocator.alloc(u32, blocks.len / blocks_per_slot); + // Slot 0 first. + for (free, 0..) |*slot, index| slot.* = @intCast(free.len - 1 - index); + return .{ .blocks = blocks, .blocks_per_slot = blocks_per_slot, .free = free, .free_len = free.len }; + } + + fn deinit(self: *Credits, allocator: std.mem.Allocator) void { + std.debug.assert(self.free_len == self.free.len); + allocator.free(self.free); + } + + fn count(self: *const Credits) usize { + return self.free.len; + } + + /// Waits for a free slot. Requests in flight give theirs back even after + /// a failure, so a waiter always gets one. + fn acquire(self: *Credits, io: std.Io) u32 { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (self.free_len == 0) { + self.condition.waitUncancelable(io, &self.mutex); + } + self.free_len -= 1; + self.high_water = @max(self.high_water, self.free.len - self.free_len); + return self.free[self.free_len]; + } + + fn release(self: *Credits, io: std.Io, slot: u32) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + std.debug.assert(self.free_len < self.free.len); + self.free[self.free_len] = slot; + self.free_len += 1; + // One release frees one slot: waking every waiter here would be a + // thundering herd. + self.condition.signal(io); + } + + fn slotBlocks(self: *const Credits, slot: u32) []const host_memory.Block { + return self.blocks[slot * self.blocks_per_slot ..][0..self.blocks_per_slot]; + } + + fn inUse(self: *Credits, io: std.Io) usize { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + return self.free.len - self.free_len; + } +}; + /// A value initialized at most once by whichever task touches it first, /// with the initializer that task passes. Concurrent callers wait on the /// event; a failed initialization keeps its error code and re-materializes From f6fef1ea0d02f1593ddd037b8cca92f819cb4abb Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:02:47 +0100 Subject: [PATCH 067/103] zml/io: add DestinationState for device memory A destination's device state is one PJRT host-to-device transfer manager and buffer per device of its grid, in canonical order, created once through LazyOnce by whichever worker first reads for it. Creating them also writes the caller's output buffer, once. On failure, the managers created so far are destroyed with their buffers. A Target tracks one device buffer's submitted bytes against its total. PJRT readies a buffer once every transfer to it has completed and one carried the last-transfer flag, whatever their order, so the flag goes on the submission that completes the buffer's bytes, never on the highest-offset piece: flagging by offset makes tails wait for their prefixes, which can deadlock a sharded load whose tails hold every credit while their prefixes stay unclaimed. This follows XLA's CommonAsyncHostToDeviceTransferManager. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 130 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 7e53a648..1adb3044 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -1,6 +1,9 @@ const std = @import("std"); +const pjrt = @import("pjrt"); + const Buffer = @import("../buffer.zig").Buffer; +const pjrtx = @import("../pjrtx.zig"); const Platform = @import("../platform.zig").Platform; const safetensors = @import("../safetensors.zig"); const Shape = @import("../shape.zig").Shape; @@ -8,8 +11,16 @@ const backend = @import("backend.zig"); const ShardGrid = @import("ShardGrid.zig"); const host_memory = @import("host_memory.zig"); +/// Creating a tensor's device state: host memory or PJRT, since the front +/// end checked its placement. `LazyOnce` replays it to every later reader +/// of the tensor. +const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; + /// One output buffer and what places it. Its parts may come from several -/// files, so several plans can write it. +/// files, so several plans can write it: their pieces all count toward the +/// same targets (`Target.total` is the placement's bytes, whichever plan +/// submits them), and its device state is created once, by the first +/// worker that reads for it. const Destination = struct { /// The caller's shape, which the output buffer keeps. shape: Shape, @@ -20,6 +31,15 @@ const Destination = struct { /// What fills it, in ascending, disjoint destination order /// (`backend.LoadSpec`). parts: []Part, + state: LazyOnce(DestinationState) = .{}, + + fn ensureState(self: *Destination, allocator: std.mem.Allocator, io: std.Io, platform: *const Platform) StateInitError!*DestinationState { + return self.state.ensure(io, DestinationState.init, .{ allocator, platform, self }); + } + + fn deinit(self: *Destination, allocator: std.mem.Allocator, api: *const pjrt.Api) void { + if (self.state.readyValue()) |state| state.deinit(allocator, api); + } }; /// One source range of a destination: what the planner sorts by file offset @@ -417,6 +437,97 @@ fn readFilePositionalAllV( return completed; } +const DestinationState = struct { + /// One destination buffer of a tensor. PJRT makes the buffer ready once + /// every transfer submitted to it has completed and one of them carried + /// the last-transfer flag, whatever their completion order: the flag + /// only closes the buffer to further calls. The pump, the only + /// submitter, therefore flags the submission that completes the + /// placement's bytes, and no piece ever waits for another one. + /// Flagging the highest-offset piece instead makes a tail wait for its + /// prefixes, which can deadlock a sharded multi-device load: tails can + /// hold every lifecycle credit while their prefixes stay unclaimed, + /// whatever the pinned memory. This follows XLA's + /// CommonAsyncHostToDeviceTransferManager: readiness requires zero + /// in-flight transfers and a last-call flag, not destination-offset order. + const Target = struct { + manager: *pjrt.AsyncHostToDeviceTransferManager, + device_index: usize, + /// Bytes of the placement on this device. The pieces partition them: + /// those of every part, from whichever file and plan. + total: usize, + /// Owned by the pump; read by the awaiting task once the batch is + /// done and nothing submits any more. + submitted_bytes: usize = 0, + + /// Whether a submission of `len` bytes closes the buffer. + fn nextIsLast(self: *const Target, len: usize) bool { + std.debug.assert(len != 0 and self.submitted_bytes + len <= self.total); + return self.submitted_bytes + len == self.total; + } + + fn noteSubmitted(self: *Target, len: usize) void { + std.debug.assert(self.submitted_bytes + len <= self.total); + self.submitted_bytes += len; + } + + /// Every byte went out, so the last-transfer flag did too. + fn fullySubmitted(self: *const Target) bool { + return self.submitted_bytes == self.total; + } + }; + + /// The destination's transfer targets, one per device in canonical + /// order. + targets: []Target, + + /// Creates the destination's device buffers and transfer managers, one + /// per device of its grid in canonical order, and writes the output + /// shell. Runs once per destination (`LazyOnce`), however many parts + /// feed it, on the worker that first reads for it. + fn init(allocator: std.mem.Allocator, platform: *const Platform, destination: *const Destination) StateInitError!DestinationState { + const grid = &destination.grid; + const targets = try allocator.alloc(Target, grid.devices.len); + errdefer allocator.free(targets); + + var pjrt_buffers: Buffer.Shards = .empty; + var initialized: usize = 0; + errdefer { + for (targets[0..initialized]) |target| target.manager.deinit(platform.pjrt_api); + for (pjrt_buffers.constSlice()) |buffer| buffer.deinit(platform.pjrt_api); + } + + const shape_spec: pjrt.ShapeSpec = .init( + grid.local.dims(), + pjrtx.bufferTypeFromDtype(grid.local.dtype()), + ); + for (grid.devices, 0..) |device, i| { + const memory = platform.devices[device.id].memory(.default).?; + const manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ + .shape_specs = &.{shape_spec}, + .memory = memory.pjrt_memory, + }); + errdefer manager.deinit(platform.pjrt_api); + const pjrt_buffer = try manager.retrieveBuffer(platform.pjrt_api, 0); + targets[i] = .{ + .manager = manager, + .device_index = device.id, + .total = grid.local.byteSize(), + }; + initialized += 1; + pjrt_buffers.appendAssumeCapacity(pjrt_buffer); + } + + destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); + return .{ .targets = targets }; + } + + fn deinit(self: *DestinationState, allocator: std.mem.Allocator, api: *const pjrt.Api) void { + for (self.targets) |target| target.manager.deinit(api); + allocator.free(self.targets); + } +}; + /// The lifecycle credits: one per request the pre-grown pinned set holds, /// each a fixed slot of `blocks_per_slot` blocks. A worker takes a credit /// before claiming a job, and the request gives it back with its last @@ -660,6 +771,23 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per try std.testing.expectEqual(@as(usize, 5), aligned_plan.jobs[2].kind.dma_read.len); } +// DMA pump and transfer target tests. + +test "the submission that completes a target's bytes carries the last flag" { + var target: DestinationState.Target = .{ .manager = undefined, .device_index = 0, .total = 100 }; + try std.testing.expect(!target.fullySubmitted()); + // Pieces arrive in any order; only the byte total matters. + try std.testing.expect(!target.nextIsLast(20)); + target.noteSubmitted(20); + try std.testing.expect(!target.nextIsLast(30)); + target.noteSubmitted(30); + try std.testing.expect(!target.nextIsLast(10)); + try std.testing.expect(target.nextIsLast(50)); + try std.testing.expect(!target.fullySubmitted()); + target.noteSubmitted(50); + try std.testing.expect(target.fullySubmitted()); +} + // Source read tests. test "readFilePositionalAllV resumes short reads and stops at minimum" { From 1f2c70076cef8cbd528a810bc66eaf9eb50fe346 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:05:25 +0100 Subject: [PATCH 068/103] zml/io: add direct batches and the FIFO scheduler A Batch is one submission to the direct backend: its plans, one per source file, and the destinations and parts they write, all in the batch's arena. It counts completion units, one per published job plus a publish sentinel held until the submission is sealed. Each job's unit is released exactly once, by its request's last reference or by a failure that retires it unclaimed; the last one sets the batch's done event, and releasing a unit is the releaser's last access to the batch. The Scheduler is a strict FIFO of published plans. A plan, its completion units and its queue entry appear in one critical section, so a queued plan's batch always holds a unit and is never freed under the scheduler, and workers need no rendezvous with the awaiting task. Jobs are claimed in plan order, a plan leaving the queue with its last claim. fail stops claims and publishes, wakes waiting workers and retires every queued plan's unclaimed units, so a failed batch still completes through the requests already claimed. Planner.publishFiles plans and publishes one file at a time, so the first file can be read while the rest is planned: planning the whole model first delayed DeepSeek's first read by 0.32 s on one MI300X. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 480 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 465 insertions(+), 15 deletions(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 1adb3044..06f05242 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -16,6 +16,66 @@ const host_memory = @import("host_memory.zig"); /// of the tensor. const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; +/// One submission: its plans (one per source file, published as each file +/// is planned), the destinations it writes and their parts, all in the +/// batch's arena until it retires. `remaining` counts completion units: one +/// per published job plus a publish sentinel held until the submission is +/// sealed. A job's unit is released exactly once, by whichever of these +/// happens: its request's last reference drops (final DMA callback or +/// abandonment), or `Scheduler.fail` retires it unclaimed. The batch is +/// done when `remaining` reaches zero; the awaiting task then retires it, +/// so releasing a unit is the last permitted access to the batch. +pub const Batch = struct { + /// Holds the plans, the destinations and the parts until `destroy`; + /// only the submitting task allocates in it. + arena: std.heap.ArenaAllocator, + io: std.Io, + /// Published plans in file order: appended by the submitting task under + /// the scheduler mutex. + plans: std.ArrayList(*Plan) = .empty, + /// Their device state is the loader's, released before `destroy`. + destinations: []Destination = &.{}, + /// The destinations' parts, each destination's consecutive and in its + /// destination order: what the planner sorts by file offset. + parts: []Part = &.{}, + remaining: std.atomic.Value(usize), + done: std.Io.Event = .unset, + /// Submission number within the loader, for log correlation. + sequence: usize, + + /// An open batch holding its publish sentinel and nothing else. The + /// sentinel keeps the batch from completing before its last plan is + /// visible. + fn create(allocator: std.mem.Allocator, io: std.Io, sequence: usize) !*Batch { + const self = try allocator.create(Batch); + self.* = .{ + .arena = .init(allocator), + .io = io, + .remaining = .init(1), + .sequence = sequence, + }; + return self; + } + + /// Releases `count` completion units. MEMORY-ORDER RULE: this must be the + /// caller's final access to the batch and to anything it owns (plans, + /// destinations, parts). The last unit sets `done`, and the awaiting + /// task frees all of it as soon as it observes the event. + fn releaseUnits(self: *Batch, count: usize) void { + if (count == 0) return; + const previous = self.remaining.fetchSub(count, .acq_rel); + std.debug.assert(previous >= count); + if (previous == count) self.done.set(self.io); + } + + /// Frees the arena: the plans, the destinations and the parts. + fn destroy(self: *Batch) void { + const allocator = self.arena.child_allocator; + self.arena.deinit(); + allocator.destroy(self); + } +}; + /// One output buffer and what places it. Its parts may come from several /// files, so several plans can write it: their pieces all count toward the /// same targets (`Target.total` is the placement's bytes, whichever plan @@ -111,12 +171,17 @@ const Plan = struct { len: usize, }; + /// The submission this plan belongs to; set when it is published, + /// under the scheduler mutex, with its completion units. + batch: *Batch = undefined, /// The file every job of the plan reads. source_slot: *SourceSlot, jobs: []Job, transfers: []Transfer, /// The tensor bytes the jobs read. source_bytes: u64, + /// Next job to claim; owned by the scheduler mutex. + cursor: usize = 0, fn transfersOf(self: *const Plan, job: Job) []const Transfer { return self.transfers[job.transfer_start..][0..job.transfer_len]; @@ -140,6 +205,30 @@ const Planner = struct { request_size: usize, }; + /// Plans `parts` one file at a time and publishes each plan as soon as + /// it exists, so workers claim the first file while the rest is planned, + /// each plan in the batch's arena. Stops at the first error; the caller + /// seals or fails the batch. + /// Whole-model planning delayed the first read by 0.32 s for DeepSeek's + /// 46 files on one MI300X; publishing per file overlaps the rest with IO. + fn publishFiles( + scheduler: *Scheduler, + io: std.Io, + batch: *Batch, + parts: []Part, + config: Config, + ) !void { + const order = try sortedPartOrder(scheduler.allocator, parts); + defer scheduler.allocator.free(order); + var file_start: usize = 0; + while (file_start < order.len) { + const file_end = fileGroupEnd(parts, order, file_start); + const plan = try preparePlan(scheduler.allocator, batch.arena.allocator(), parts, order[file_start..file_end], config); + try scheduler.publish(io, batch, plan); + file_start = file_end; + } + } + /// Plans one file: `order` indexes its `parts` by offset and size /// (`sortedPartOrder`), and `JobCut.cutRuns` cuts them. The plan goes in /// `arena` (`JobCut.finish`), the cut in `allocator`. @@ -407,6 +496,108 @@ const Planner = struct { } }; +/// A strict FIFO of published plans. A submission holds one plan per source +/// file, published as soon as that file is planned, so a plan reaches the +/// queue behind every earlier plan of its own submission and of every +/// earlier one (submissions publish and seal on one task). Within a plan, +/// jobs are handed out in the planned order (file order); a plan leaves the +/// queue with its last claim or through `fail`. +/// A plan is queued only with jobs left and its completion units are added +/// in the same critical section, so a queued plan's batch still holds at +/// least one unit and can never be freed under the scheduler. +const Scheduler = struct { + /// A claimed job: its position in the plan that owns it. The claim holds + /// one of the batch's completion units until the request releases it. + const Claim = struct { + plan: *Plan, + index: usize, + }; + + allocator: std.mem.Allocator, + /// Plans with unclaimed jobs in publish order. + queue: std.Deque(*Plan) = .empty, + /// Set by `fail`: `publish` refuses plans (`Canceled`) and `waitForWork` + /// returns false. + stopping: bool = false, + mutex: std.Io.Mutex = .init, + condition: std.Io.Condition = .init, + + fn init(allocator: std.mem.Allocator) Scheduler { + return .{ .allocator = allocator }; + } + + fn deinit(self: *Scheduler) void { + // Every batch was awaited (and so claimed out or retired) first. + std.debug.assert(self.queue.len == 0); + self.queue.deinit(self.allocator); + self.* = undefined; + } + + /// Publishes one plan of an open submission behind every earlier plan. + /// A plan without jobs never joins the queue. `Canceled` once the + /// scheduler failed (`fail`). + fn publish(self: *Scheduler, io: std.Io, batch: *Batch, plan: *Plan) !void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + if (self.stopping) return error.Canceled; + const job_count = plan.jobs.len; + try batch.plans.ensureUnusedCapacity(batch.arena.allocator(), 1); + if (job_count != 0) try self.queue.ensureUnusedCapacity(self.allocator, 1); + // Nothing below fails: the plan, its units and the queue entry + // appear together. + plan.batch = batch; + batch.plans.appendAssumeCapacity(plan); + _ = batch.remaining.fetchAdd(job_count, .acq_rel); + if (job_count != 0) { + self.queue.pushBackAssumeCapacity(plan); + self.condition.broadcast(io); + } + } + + /// Stops claims and publishes, wakes every worker waiting for work, and + /// retires the unclaimed units of every queued plan so its batch still + /// reaches `done` through its claimed requests (an open one through its + /// seal and sentinel as well). Claims and this pass both move cursors + /// under the mutex, so they partition the jobs exactly: every claimed + /// job keeps its unit with its worker. Each queued plan still holds an + /// unclaimed unit of its batch, so only the retire of a batch's last + /// queued plan can complete it, and nothing touches the batch after that. + fn fail(self: *Scheduler, io: std.Io) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.stopping = true; + self.condition.broadcast(io); + while (self.queue.popFront()) |plan| { + const unclaimed = plan.jobs.len - plan.cursor; + plan.cursor = plan.jobs.len; + plan.batch.releaseUnits(unclaimed); + } + } + + /// Hands out the head plan's next job; the plan leaves the queue with + /// its last one. Null only when nothing is queued: a queued plan always + /// has a job left. + fn claim(self: *Scheduler, io: std.Io) ?Claim { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + const plan = self.queue.front() orelse return null; + const index = plan.cursor; + std.debug.assert(index < plan.jobs.len); + plan.cursor += 1; + if (plan.cursor == plan.jobs.len) _ = self.queue.popFront(); + return .{ .plan = plan, .index = index }; + } + + fn waitForWork(self: *Scheduler, io: std.Io) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (!self.stopping and self.queue.len == 0) { + self.condition.waitUncancelable(io, &self.mutex); + } + return !self.stopping; + } +}; + /// Reads `[file_offset, file_offset + minimum)`, and possibly more, into /// `buffers` in order, consuming them as it reads; bytes past `minimum` may /// stay unread. The io takes as many buffers per call as it can (VFS's @@ -687,6 +878,20 @@ fn initPlainParts( } } +/// Claims out and frees an open batch that a test only inspected; it must +/// be the only queued batch. +fn discardTestBatch(scheduler: *Scheduler, batch: *Batch) void { + const io = std.testing.io; + var claimed: usize = 0; + while (scheduler.claim(io)) |claim| { + std.debug.assert(claim.plan.batch == batch); + claimed += 1; + } + batch.releaseUnits(1 + claimed); + std.debug.assert(batch.done.isSet()); + batch.destroy(); +} + test "source planner coalesces exact adjacent and overlapping tensor ranges per file" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -710,25 +915,19 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per var parts: [sources.len]Part = undefined; initPlainParts(platform, &sources, &.{ &slots[0], &slots[0], &slots[0], &slots[0], &slots[1] }, &outputs, &destinations, &parts); - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const order = try Planner.sortedPartOrder(allocator, &parts); - defer allocator.free(order); - var plans: [2]*Plan = undefined; - var file_start: usize = 0; - for (&plans) |*plan| { - const file_end = Planner.fileGroupEnd(&parts, order, file_start); - plan.* = try prepareTestPlan(arena.allocator(), &parts, order[file_start..file_end], .{ - .block_size = 4, - .request_size = 8, - }); - file_start = file_end; - } - try std.testing.expectEqual(order.len, file_start); + var scheduler: Scheduler = .init(allocator); + defer scheduler.deinit(); + const batch = try Batch.create(allocator, io, 0); + try Planner.publishFiles(&scheduler, io, batch, &parts, .{ + .block_size = 4, + .request_size = 8, + }); // One plan per file. a:[10,18) merges adjacency and the duplicate, // a:[20,24) remains exact, and b:[3,15) is split at the request-size // boundary. + const plans = batch.plans.items; + try std.testing.expectEqual(@as(usize, 2), plans.len); try std.testing.expect(plans[0].source_slot == &slots[0]); try std.testing.expect(plans[1].source_slot == &slots[1]); try std.testing.expectEqual(@as(usize, 2), plans[0].jobs.len); @@ -749,6 +948,7 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per try std.testing.expectEqual(@as(usize, 1), plans[0].jobs[1].blockLen(4)); try std.testing.expectEqual(@as(usize, 2), plans[1].jobs[0].blockLen(4)); try std.testing.expectEqual(@as(usize, 1), plans[1].jobs[1].blockLen(4)); + discardTestBatch(&scheduler, batch); var aligned_sources = [_]safetensors.Tensor{ .{ .file_uri = "aligned", .name = "aligned0", .shape = .init(.{7}, .u8), .offset = 0 }, @@ -760,6 +960,8 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per var aligned_destinations: [aligned_sources.len]Destination = undefined; var aligned_items: [aligned_sources.len]Part = undefined; initPlainParts(platform, &aligned_sources, &.{ &aligned_slot, &aligned_slot, &aligned_slot }, &aligned_outputs, &aligned_destinations, &aligned_items); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); const aligned_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ .block_size = 4, .request_size = 8, @@ -771,6 +973,254 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per try std.testing.expectEqual(@as(usize, 5), aligned_plan.jobs[2].kind.dma_read.len); } +// Scheduler tests. + +/// A plan of `job_count` unit jobs (`file_offset` = index), each with one +/// block, and no transfers, in `arena`. +fn testPlan(arena: std.mem.Allocator, job_count: usize) !*Plan { + const jobs = try arena.alloc(Plan.Job, job_count); + for (jobs, 0..) |*job, index| job.* = .{ + .file_offset = index, + .transfer_start = 0, + .transfer_len = 0, + .kind = .{ .dma_read = .{ .len = 1, .minimum_len = 1 } }, + }; + const plan = try arena.create(Plan); + plan.* = .{ + .source_slot = undefined, + .jobs = jobs, + .transfers = &.{}, + .source_bytes = job_count, + }; + return plan; +} + +/// Publishes a plan of `job_count` unit jobs into an open batch. +fn publishTestPlan(scheduler: *Scheduler, batch: *Batch, job_count: usize) !void { + try scheduler.publish(std.testing.io, batch, try testPlan(batch.arena.allocator(), job_count)); +} + +/// A one-file submission's sequence: publish, then drop the sentinel. +fn publishTestBatch(scheduler: *Scheduler, job_count: usize) !*Batch { + const batch = try Batch.create(std.testing.allocator, std.testing.io, 0); + errdefer batch.destroy(); + try publishTestPlan(scheduler, batch, job_count); + batch.releaseUnits(1); + return batch; +} + +/// The jobs of the queued plans no worker claimed yet. +fn unclaimedJobs(scheduler: *const Scheduler) usize { + var jobs: usize = 0; + for (0..scheduler.queue.len) |index| { + const plan = scheduler.queue.at(index); + jobs += plan.jobs.len - plan.cursor; + } + return jobs; +} + +test "scheduler publishes a submission one file at a time and claims the files in order" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + // Submitted out of file order: the sort groups "a" before "b". + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "b", .name = "b0", .shape = .init(.{4}, .u8), .offset = 0 }, + .{ .file_uri = "a", .name = "a1", .shape = .init(.{4}, .u8), .offset = 4 }, + .{ .file_uri = "a", .name = "a0", .shape = .init(.{4}, .u8), .offset = 0 }, + }; + var slots = [_]SourceSlot{ + .{ .uri = "a" }, + .{ .uri = "b" }, + }; + var outputs: [sources.len]Buffer = undefined; + var destinations: [sources.len]Destination = undefined; + var parts: [sources.len]Part = undefined; + initPlainParts(platform, &sources, &.{ &slots[1], &slots[0], &slots[0] }, &outputs, &destinations, &parts); + + var scheduler: Scheduler = .init(allocator); + defer scheduler.deinit(); + const batch = try Batch.create(allocator, io, 0); + try Planner.publishFiles(&scheduler, io, batch, &parts, .{ + .block_size = 4, + .request_size = 4, + }); + try std.testing.expectEqual(@as(usize, 2), batch.plans.items.len); + try std.testing.expectEqual(@as(usize, 2), batch.plans.items[0].jobs.len); + try std.testing.expectEqual(@as(usize, 1), batch.plans.items[1].jobs.len); + try std.testing.expectEqual(@as(usize, 3), unclaimedJobs(&scheduler)); + + // File a's jobs in offset order, then file b's. + var claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan.source_slot == &slots[0]); + try std.testing.expectEqual(@as(u64, 0), claim.plan.jobs[claim.index].file_offset); + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan.source_slot == &slots[0]); + try std.testing.expectEqual(@as(u64, 4), claim.plan.jobs[claim.index].file_offset); + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan.source_slot == &slots[1]); + try std.testing.expectEqual(@as(u64, 0), claim.plan.jobs[claim.index].file_offset); + // Both plans left the queue with their last claim; the submission is + // still open until its sentinel goes. + try std.testing.expect(scheduler.claim(io) == null); + try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); + batch.releaseUnits(1); + try std.testing.expect(!batch.done.isSet()); + batch.releaseUnits(3); + try std.testing.expect(batch.done.isSet()); + batch.destroy(); +} + +test "fifo scheduler claims plan by plan in publish order, and a done batch leaves while a later plan is queued" { + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + // An open submission's first plan, a sealed batch, then the open + // submission's second plan: an open submission holds no place of its + // own. + const open = try Batch.create(std.testing.allocator, io, 0); + try publishTestPlan(&scheduler, open, 1); + const sealed = try publishTestBatch(&scheduler, 2); + try publishTestPlan(&scheduler, open, 1); + try std.testing.expectEqual(@as(usize, 4), unclaimedJobs(&scheduler)); + + var claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan == open.plans.items[0] and claim.index == 0); + for (0..2) |index| { + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan == sealed.plans.items[0] and claim.index == index); + } + // The sealed batch left the queue with its last claim, so it completes + // and goes away while the open one's second plan is still queued. + sealed.releaseUnits(2); + try std.testing.expect(sealed.done.isSet()); + sealed.destroy(); + try std.testing.expectEqual(@as(usize, 1), scheduler.queue.len); + + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan == open.plans.items[1] and claim.index == 0); + try std.testing.expect(scheduler.claim(io) == null); + try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); + // Claimed out, the open submission completes only once its sentinel + // goes. + open.releaseUnits(2); + try std.testing.expect(!open.done.isSet()); + open.releaseUnits(1); + try std.testing.expect(open.done.isSet()); + open.destroy(); +} + +test "fifo scheduler failure retires the unclaimed jobs of every queued plan and refuses later publishes" { + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + // A batch without jobs is never queued and completes at publish. + const empty = try publishTestBatch(&scheduler, 0); + try std.testing.expect(empty.done.isSet()); + empty.destroy(); + // An open submission of two plans, then a sealed batch. + const open = try Batch.create(std.testing.allocator, io, 0); + try publishTestPlan(&scheduler, open, 2); + try publishTestPlan(&scheduler, open, 3); + const sealed = try publishTestBatch(&scheduler, 3); + try std.testing.expectEqual(@as(usize, 8), unclaimedJobs(&scheduler)); + try std.testing.expectEqual(@as(usize, 6), open.remaining.load(.acquire)); + + // The claim keeps its unit and the open submission its sentinel; every + // other job is retired. + try std.testing.expect(scheduler.claim(io).?.plan.batch == open); + scheduler.fail(io); + try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); + try std.testing.expectEqual(@as(usize, 2), open.remaining.load(.acquire)); + try std.testing.expect(sealed.done.isSet()); + sealed.destroy(); + try std.testing.expect(scheduler.claim(io) == null); + try std.testing.expect(!scheduler.waitForWork(io)); + // The submission goes on: its next plan is refused, and the sentinel + // drop leaves only the claim to complete it. + try std.testing.expectError(error.Canceled, publishTestPlan(&scheduler, open, 1)); + open.releaseUnits(1); + try std.testing.expect(!open.done.isSet()); + open.releaseUnits(1); + try std.testing.expect(open.done.isSet()); + open.destroy(); +} + +test "fifo scheduler wakes waiting workers on publish and releases them on failure" { + const WaitResult = enum(u8) { waiting, work, stopped }; + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + + var woke: std.atomic.Value(WaitResult) = .init(.waiting); + var group: std.Io.Group = .init; + const Waiter = struct { + fn run(scheduler_: *Scheduler, io_: std.Io, woke_: *std.atomic.Value(WaitResult)) void { + woke_.store(if (scheduler_.waitForWork(io_)) .work else .stopped, .release); + } + }; + try group.concurrent(io, Waiter.run, .{ &scheduler, io, &woke }); + try io.sleep(.fromMilliseconds(5), .awake); + try std.testing.expectEqual(WaitResult.waiting, woke.load(.acquire)); + const batch = try publishTestBatch(&scheduler, 1); + try group.await(io); + try std.testing.expectEqual(WaitResult.work, woke.load(.acquire)); + _ = scheduler.claim(io).?; + batch.releaseUnits(1); + batch.destroy(); + + woke.store(.waiting, .release); + try group.concurrent(io, Waiter.run, .{ &scheduler, io, &woke }); + try io.sleep(.fromMilliseconds(5), .awake); + try std.testing.expectEqual(WaitResult.waiting, woke.load(.acquire)); + scheduler.fail(io); + try group.await(io); + try std.testing.expectEqual(WaitResult.stopped, woke.load(.acquire)); +} + +test "fifo scheduler concurrent claims across two batches return each job once" { + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + const batches = [_]*Batch{ + try publishTestBatch(&scheduler, 16), + try publishTestBatch(&scheduler, 16), + }; + var seen: std.atomic.Value(u64) = .init(0); + var claim_count: std.atomic.Value(usize) = .init(0); + var duplicate: std.atomic.Value(bool) = .init(false); + var group: std.Io.Group = .init; + for (0..8) |_| try group.concurrent(io, struct { + fn run( + scheduler_: *Scheduler, + batches_: []const *Batch, + seen_: *std.atomic.Value(u64), + claim_count_: *std.atomic.Value(usize), + duplicate_: *std.atomic.Value(bool), + ) void { + while (scheduler_.claim(std.testing.io)) |claim| { + const base: u64 = if (claim.plan.batch == batches_[0]) 0 else 16; + const mask = @as(u64, 1) << @intCast(base + claim.plan.jobs[claim.index].file_offset); + if (seen_.fetchOr(mask, .acq_rel) & mask != 0) duplicate_.store(true, .release); + _ = claim_count_.fetchAdd(1, .monotonic); + } + } + }.run, .{ &scheduler, &batches, &seen, &claim_count, &duplicate }); + try group.await(io); + try std.testing.expectEqual(std.math.maxInt(u32), @as(u32, @truncate(seen.load(.acquire)))); + try std.testing.expectEqual(@as(usize, 32), claim_count.load(.acquire)); + try std.testing.expect(!duplicate.load(.acquire)); + try std.testing.expectEqual(@as(usize, 0), unclaimedJobs(&scheduler)); + for (batches) |batch| { + batch.releaseUnits(16); + try std.testing.expect(batch.done.isSet()); + batch.destroy(); + } +} + // DMA pump and transfer target tests. test "the submission that completes a target's bytes carries the last flag" { From e03d7b12d32142eb791c7298b540eb0d2c9f312e Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:10:11 +0100 Subject: [PATCH 069/103] zml/io: add Plan's request and event arrays and ReadRequest A plan now allocates every context its jobs can need before it is published: one ReadRequest per job, idle until a worker claims the job, and one event context per planned DMA submission, a writer of a transfer each. The batch's arena keeps them until the batch retires, so callbacks can hold their addresses. A ReadRequest is a claimed job with the worker's lifecycle credit. Its run creates the device state of the job's destinations if no worker has, reads the job into its slot's blocks in file order, counts the bytes read toward each destination and queues the job's transfer pieces for DMA. It holds a scheduling reference and one per queued piece; the last release gives the credit back, then releases the batch's completion unit, which may free the batch, so nothing is touched after it. This and the next commit refer to the DMA stage and the loader that the runtime commit adds: the three are mutually recursive and build together. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 176 +++++++++++++++++++++++++++++++++++---- 1 file changed, 161 insertions(+), 15 deletions(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 06f05242..4bf90e86 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -17,8 +17,9 @@ const host_memory = @import("host_memory.zig"); const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; /// One submission: its plans (one per source file, published as each file -/// is planned), the destinations it writes and their parts, all in the -/// batch's arena until it retires. `remaining` counts completion units: one +/// is planned), the destinations it writes and their parts, and every +/// request and event context created while loading it, all in the batch's +/// arena until it retires. `remaining` counts completion units: one /// per published job plus a publish sentinel held until the submission is /// sealed. A job's unit is released exactly once, by whichever of these /// happens: its request's last reference drops (final DMA callback or @@ -26,8 +27,8 @@ const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; /// done when `remaining` reaches zero; the awaiting task then retires it, /// so releasing a unit is the last permitted access to the batch. pub const Batch = struct { - /// Holds the plans, the destinations and the parts until `destroy`; - /// only the submitting task allocates in it. + /// Holds the plans with their contexts, the destinations and the parts + /// until `destroy`; only the submitting task allocates in it. arena: std.heap.ArenaAllocator, io: std.Io, /// Published plans in file order: appended by the submitting task under @@ -58,9 +59,11 @@ pub const Batch = struct { } /// Releases `count` completion units. MEMORY-ORDER RULE: this must be the - /// caller's final access to the batch and to anything it owns (plans, - /// destinations, parts). The last unit sets `done`, and the awaiting - /// task frees all of it as soon as it observes the event. + /// caller's final access to the batch and to anything it owns (requests, + /// events, destinations, parts, plans; a request's slot of blocks is the + /// loader's and went back with its credit just before). The last unit + /// sets `done`, and the awaiting task frees all of it as soon as it + /// observes the event. fn releaseUnits(self: *Batch, count: usize) void { if (count == 0) return; const previous = self.remaining.fetchSub(count, .acq_rel); @@ -125,11 +128,14 @@ const SourceSlot = struct { uri: []const u8, }; -/// One file's jobs in claim order with their transfer records, allocated by -/// the planner. +/// One file's jobs in claim order with their transfer records and every +/// context the jobs can need, allocated by the planner: one request per job +/// and one event per planned DMA submission, handed out in submission +/// order. Contexts hold the plan's address, which the batch's arena keeps +/// until it retires. const Plan = struct { /// A claimable job: its slice of the plan's transfers and what its - /// worker does for them. + /// worker does for them. Its request slot is its index. const Job = struct { /// Where the job's read starts. file_offset: u64, @@ -178,6 +184,11 @@ const Plan = struct { source_slot: *SourceSlot, jobs: []Job, transfers: []Transfer, + requests: []ReadRequest, + events: []Dma.EventContext, + /// Event slots handed out so far, one `fetchAdd` per submission from + /// any device's pump. + events_used: std.atomic.Value(usize) = .init(0), /// The tensor bytes the jobs read. source_bytes: u64, /// Next job to claim; owned by the scheduler mutex. @@ -272,13 +283,22 @@ const Planner = struct { safe_boundaries: std.ArrayList(u64) = .empty, source_bytes: u64 = 0, - /// The plan of the jobs cut, copied into `arena`. + /// The plan of the jobs cut, copied into `arena` with every context + /// its jobs can need, allocated before publication makes it visible + /// to workers: a request per job and an event per writer of a + /// transfer. fn finish(self: *JobCut, arena: std.mem.Allocator) !*Plan { + const jobs = self.jobs.items; + const transfers = self.transfers.items; + const requests = try arena.alloc(ReadRequest, jobs.len); + @memset(requests, ReadRequest.idle); const plan = try arena.create(Plan); plan.* = .{ .source_slot = self.source_slot, - .jobs = try arena.dupe(Plan.Job, self.jobs.items), - .transfers = try arena.dupe(Plan.Transfer, self.transfers.items), + .jobs = try arena.dupe(Plan.Job, jobs), + .transfers = try arena.dupe(Plan.Transfer, transfers), + .requests = requests, + .events = try arena.alloc(Dma.EventContext, submissionCount(transfers)), .source_bytes = self.source_bytes, }; return plan; @@ -457,6 +477,12 @@ const Planner = struct { }); } } + + fn submissionCount(transfers: []const Plan.Transfer) usize { + var count: usize = 0; + for (transfers) |transfer| count += @popCount(transfer.writer_mask); + return count; + } }; fn blockCount(len: usize, block_size: usize) usize { return len / block_size + @intFromBool(len % block_size != 0); @@ -598,6 +624,114 @@ const Scheduler = struct { } }; +/// A claimed source job and its completion state. The plan owns its storage. +const ReadRequest = struct { + const Scratch = struct { + allocator: std.mem.Allocator, + /// A read's ranges: one per block of a job (`readBlocks`). + iovecs: [][]u8, + + fn init(allocator: std.mem.Allocator, maximum_blocks: usize) !Scratch { + return .{ + .allocator = allocator, + .iovecs = try allocator.alloc([]u8, maximum_blocks), + }; + } + + fn deinit(self: *Scratch) void { + self.allocator.free(self.iovecs); + self.* = undefined; + } + }; + + loader: *Loader, + plan: *Plan, + /// The job's index in its plan, which holds this request at the same + /// index. + index: usize, + /// The request's lifecycle credit: its slot of blocks (`Credits`), which + /// only this request reads into and transfers from until it ends. + slot: u32, + /// The scheduling sentinel and one reference per DMA piece queued; the + /// last release gives the credit back and finishes the job. + pending: std.atomic.Value(usize) = .init(1), + + /// The slot of a job that was never claimed: nothing pending, so the + /// retirement checks hold. + const idle: ReadRequest = .{ + .loader = undefined, + .plan = undefined, + .index = undefined, + .slot = undefined, + .pending = .init(0), + }; + + /// Takes the claimed job's request context with the worker's credit. + /// The claim holds the batch's completion unit; the request's final + /// reference drop releases it. + fn init(loader: *Loader, claim: Scheduler.Claim, slot: u32) *ReadRequest { + const self = &claim.plan.requests[claim.index]; + self.* = .{ + .loader = loader, + .plan = claim.plan, + .index = claim.index, + .slot = slot, + }; + return self; + } + + fn run(self: *ReadRequest, scratch: *Scratch) !void { + const loader = self.loader; + const block_size = loader.plan_config.block_size; + const job = self.plan.jobs[self.index]; + const transfers = self.plan.transfersOf(job); + // The first worker to touch a destination creates its state. + var previous: ?*Destination = null; + for (transfers) |transfer| { + const destination = transfer.destination; + if (destination == previous) continue; + previous = destination; + _ = destination.ensureState(loader.allocator, loader.io, loader.platform) catch |err| { + loader.recordError(err, "initialize tensor {s} from {s}, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); + return err; + }; + } + const blocks = loader.credits.slotBlocks(self.slot)[0..job.blockLen(block_size)]; + switch (job.kind) { + .dma_read => |read| try self.readBlocks(scratch, blocks, read), + } + for (transfers) |transfer| transfer.destination.recordRead(transfer.len); + try Dma.enqueue(self, blocks); + } + + /// Reads the job's `read` into `blocks`, its slot's, in file order. + fn readBlocks(self: *ReadRequest, scratch: *Scratch, blocks: []const host_memory.Block, read: Plan.Job.Read) !void { + const loader = self.loader; + const block_size = loader.plan_config.block_size; + const iovecs = scratch.iovecs[0..blocks.len]; + for (iovecs, blocks, 0..) |*iovec, block, block_index| { + iovec.* = block[0..@min(block_size, read.len - block_index * block_size)]; + } + try readSlot(loader, self.plan.source_slot, iovecs, self.plan.jobs[self.index].file_offset, read.minimum_len); + } + + /// Releases a DMA piece or the scheduling reference. The final release + /// gives the credit back and may let the awaiting task free this request + /// and everything its batch owns. + fn release(self: *ReadRequest) void { + const previous = self.pending.fetchSub(1, .acq_rel); + std.debug.assert(previous > 0); + if (previous != 1) return; + + // Locals first: releasing the batch unit may complete the batch + // that owns this request, so nothing is touched after it. + const loader = self.loader; + const batch = self.plan.batch; + loader.credits.release(loader.io, self.slot); + batch.releaseUnits(1); + } +}; + /// Reads `[file_offset, file_offset + minimum)`, and possibly more, into /// `buffers` in order, consuming them as it reads; bytes past `minimum` may /// stay unread. The io takes as many buffers per call as it can (VFS's @@ -943,11 +1077,18 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per try std.testing.expectEqual(@as(usize, 4), plans[1].jobs[1].kind.dma_read.len); try std.testing.expectEqual(@as(u64, 12), plans[0].source_bytes); try std.testing.expectEqual(@as(u64, 12), plans[1].source_bytes); - // Jobs span their 4-byte blocks. + // One writer per transfer on one device: one event per transfer. Jobs + // span their 4-byte blocks. + try std.testing.expectEqual(@as(usize, 4), plans[0].events.len); + try std.testing.expectEqual(@as(usize, 3), plans[1].events.len); + try std.testing.expectEqual(@as(usize, 2), plans[0].requests.len); try std.testing.expectEqual(@as(usize, 2), plans[0].jobs[0].blockLen(4)); try std.testing.expectEqual(@as(usize, 1), plans[0].jobs[1].blockLen(4)); try std.testing.expectEqual(@as(usize, 2), plans[1].jobs[0].blockLen(4)); try std.testing.expectEqual(@as(usize, 1), plans[1].jobs[1].blockLen(4)); + for (plans) |plan| { + for (plan.requests) |*request| try std.testing.expectEqual(@as(usize, 0), request.pending.load(.acquire)); + } discardTestBatch(&scheduler, batch); var aligned_sources = [_]safetensors.Tensor{ @@ -976,9 +1117,12 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per // Scheduler tests. /// A plan of `job_count` unit jobs (`file_offset` = index), each with one -/// block, and no transfers, in `arena`. +/// block and one event slot, and no transfers, in `arena`. fn testPlan(arena: std.mem.Allocator, job_count: usize) !*Plan { const jobs = try arena.alloc(Plan.Job, job_count); + const requests = try arena.alloc(ReadRequest, job_count); + @memset(requests, ReadRequest.idle); + const events = try arena.alloc(Dma.EventContext, job_count); for (jobs, 0..) |*job, index| job.* = .{ .file_offset = index, .transfer_start = 0, @@ -990,6 +1134,8 @@ fn testPlan(arena: std.mem.Allocator, job_count: usize) !*Plan { .source_slot = undefined, .jobs = jobs, .transfers = &.{}, + .requests = requests, + .events = events, .source_bytes = job_count, }; return plan; From 0ecf6ff488ca369d2919443e34f2cb8e81f30d26 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:11:27 +0100 Subject: [PATCH 070/103] zml/io: add the direct DMA stage: pumps, budget, all-or-nothing enqueue, retirement, abortReady A request's transfer pieces go to per-device pumps. Each device has one queue and runs at most one pump at a time, on whichever thread finds work: a worker after it queued a job, or a completion callback. One submitter per device is what lets a target's last-transfer flag go by bytes. A pump submits queued pieces in arrival order while the device has room under its budget: eight blocks of bytes in flight, and at most 64 pieces, so a model of small tensors cannot flood a device with events. The piece that crosses the byte budget is admitted, so a device with room always has one in flight. Per-device pumps loaded replicated DeepSeek on four GB300 in 4.68 s, against 5.41 s with one pump for every device. enqueue counts a job's pieces per device, locks every pump in device order, checks the loader's sticky error, reserves every queue and only then queues the whole job, so a request is fully queued or not at all; queuing piece by piece cost tens of thousands of lock and pump trips per DeepSeek load. Each submission takes a preallocated event context from its plan. A completion frees its device's room, which may pump more, then hands its fired event to the device's next pump to destroy, never inside the event's own callback, then releases its request last, since that release may complete and free the batch. retireBatch destroys what is left with every pump locked. Once the loader failed, abortReady releases every queued piece, so no request waits on a pump that stopped. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 498 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 497 insertions(+), 1 deletion(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 4bf90e86..3b5e72b0 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -16,6 +16,14 @@ const host_memory = @import("host_memory.zig"); /// of the tensor. const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; +/// Bounds per-device event overhead for tiny tensors. This is not a measured +/// optimum: 64 pieces smaller than block_size / 8 cannot fill an eight-block +/// byte budget. +const max_dma_pieces_per_device: usize = 64; + +/// Each device's in-flight DMA budget, in blocks (`Dma.DevicePump.hasRoom`). +const dma_depth: usize = 8; + /// One submission: its plans (one per source file, published as each file /// is planned), the destinations it writes and their parts, and every /// request and event context created while loading it, all in the batch's @@ -71,7 +79,9 @@ pub const Batch = struct { if (previous == count) self.done.set(self.io); } - /// Frees the arena: the plans, the destinations and the parts. + /// Frees the arena: the plans with their contexts, which + /// `Dma.retireBatch` must already have retired, the destinations and + /// the parts. fn destroy(self: *Batch) void { const allocator = self.arena.child_allocator; self.arena.deinit(); @@ -762,6 +772,346 @@ fn readFilePositionalAllV( return completed; } +/// The bytes from offset `at` of `blocks` taken end to end, `block_size` +/// bytes each, up to `len` of them or the end of that block. +fn blockPiece(blocks: []const host_memory.Block, block_size: usize, at: usize, len: usize) []u8 { + const in_block = at % block_size; + return blocks[at / block_size][in_block..][0..@min(len, block_size - in_block)]; +} + +/// The DMA stage, over the loader's pumps (`Loader.pumps`): workers queue a +/// request's transfers per device, a pump submits them under the device's +/// in-flight budget, and their callbacks release the request. +const Dma = struct { + /// One DMA piece: `data`, in its request's slot, for `target` at + /// `destination_offset`. The piece holds one of its request's + /// references until its callback. Replicas share their source bytes: + /// copying them to another CPU socket first was rejected on MI300X, + /// where raw local and remote H2D both reached ~49-50 GiB/s per GPU + /// while one CPU thread copied across sockets at 5-10 GiB/s. + const Piece = struct { + target: *DestinationState.Target, + request: *ReadRequest, + data: []const u8, + destination_offset: usize, + }; + + /// Pieces ready for one device, submitted in arrival order so the + /// oldest requests and submissions complete first. Owned by the + /// device's `DevicePump.mutex`. + const ReadyQueue = std.Deque(Piece); + + /// One device's submission state. Its pump runs on whichever thread + /// finds work -- a worker after enqueueing, a ready callback after a + /// completion -- one at a time per device (`pumping`), and shares + /// nothing with the other devices' pumps. Per-device pumps load + /// replicated DeepSeek on four GB300 in 4.68 s, against 5.41 s with one + /// pump for every device, at unchanged pinned high-water. A dedicated + /// task per device took 5.0-5.4 s with a wake per completion (a futex + /// round trip per piece), or 4.4-5.1 s with half-budget wake hysteresis. + const DevicePump = struct { + mutex: std.Io.Mutex = .init, + queue: ReadyQueue = .empty, + /// In-flight DMA bytes and submissions; owned by `mutex`. + active_bytes: usize = 0, + active_pieces: usize = 0, + /// True while one thread runs this device's pump; owned by `mutex`. + /// One submitter per device is a correctness requirement: + /// `Target.nextIsLast` flags a tensor's last piece in submission + /// order. + pumping: bool = false, + /// Contexts whose callback fired, for the next pump or a batch's + /// retirement to destroy, never the event's own callback: an + /// intrusive stack through `EventContext.next_retired`, owned by + /// `mutex`. Destroying an event from the next pump instead of at its + /// batch's retirement bounds live PJRT events by the DMA width plus + /// one pump batch rather than by a submission's transfer count; + /// every shipped plugin accepted destruction outside the event's own + /// callback (16,384 fired events on two B70, CUDA and ROCm alike). + retired: ?*EventContext = null, + + /// Whether the next queued piece may go. The budget is in bytes, + /// not submissions: `dma_depth` blocks, and a piece is one tensor's + /// slice of a block, so a model of small tensors needs many more + /// pieces in flight to keep the same bytes moving: eight average + /// DeepSeek pieces occupy only ~18 MiB against the 128 MiB budget of + /// 16 MiB blocks. The piece that crosses the budget is admitted: a + /// device with room always has a piece in flight. + fn hasRoom(self: *const DevicePump, budget_bytes: usize) bool { + return self.active_bytes < budget_bytes and self.active_pieces < max_dma_pieces_per_device; + } + + /// `mutex`. Destroys every retired event. + fn destroyRetired(self: *DevicePump) void { + while (self.retired) |ctx| { + self.retired = ctx.next_retired; + ctx.next_retired = null; + ctx.destroyEvent(); + } + } + }; + + const EventContext = struct { + request: *ReadRequest, + /// Null once destroyed: by a pump or a batch's retirement after its + /// callback ran, or at once by a failed `onReady`. + pjrt_event: ?*pjrt.Event, + device_index: usize, + /// Bytes this submission holds against the device's in-flight budget. + len: usize, + /// Link of the device pump's `retired`; owned by that pump's mutex. + next_retired: ?*EventContext = null, + + /// Pump mutex, unless no callback was registered. Destroys the + /// event, once. The platform is reached through the request, and + /// only when there is something to destroy: a context whose event + /// is null never touches it. + fn destroyEvent(self: *EventContext) void { + if (self.pjrt_event) |event| event.deinit(self.request.loader.platform.pjrt_api); + self.pjrt_event = null; + } + }; + + fn lockAllPumps(loader: *Loader) void { + for (loader.pumps) |*device_pump| device_pump.mutex.lockUncancelable(loader.io); + } + + fn unlockAllPumps(loader: *Loader) void { + var index = loader.pumps.len; + while (index > 0) { + index -= 1; + loader.pumps[index].mutex.unlock(loader.io); + } + } + + /// Retires the contexts of a done batch from the awaiting task, after + /// their callbacks fired, exactly as `pjrt.Event.await` does: drains + /// every pump's `retired`, whichever batch its contexts belong to, and + /// checks that every request completed. Runs with every pump locked so + /// an `abortReady` still iterating queued entries or a pump draining its + /// `retired` cannot race the free that follows. + fn retireBatch(loader: *Loader, batch: *Batch) void { + std.debug.assert(batch.done.isSet()); + lockAllPumps(loader); + defer unlockAllPumps(loader); + for (loader.pumps) |*device_pump| device_pump.destroyRetired(); + for (batch.plans.items) |plan| { + for (plan.requests) |*request| std.debug.assert(request.pending.load(.acquire) == 0); + } + } + + /// Hands a context whose callback fired to its device's next pump. Under + /// that pump's mutex, so the batch's retirement sees it before the batch + /// is freed. + fn retireEvent(loader: *Loader, ctx: *EventContext) void { + const device_pump = &loader.pumps[ctx.device_index]; + device_pump.mutex.lockUncancelable(loader.io); + ctx.next_retired = device_pump.retired; + device_pump.retired = ctx; + device_pump.mutex.unlock(loader.io); + } + + /// Queues every piece of the request's job before pumping: queuing piece + /// by piece costs roughly 69k-79k mutex and pump trips per DeepSeek load, + /// which coalesced reads alone do not remove. A transfer reads `len` + /// bytes from its `landing` of `blocks` taken end to end, inside one + /// block, once for each of its writers: the devices the planner planned + /// for (both come from the sharding's canonical device order). + fn enqueue(request: *ReadRequest, blocks: []const host_memory.Block) !void { + const loader = request.loader; + const block_size = loader.plan_config.block_size; + const transfers = request.plan.transfersOf(request.plan.jobs[request.index]); + var counts: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + var pieces: usize = 0; + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + pieces += @popCount(transfer.writer_mask); + var mask = transfer.writer_mask; + while (mask != 0) { + const writer_index: usize = @intCast(@ctz(mask)); + mask &= mask - 1; + counts[tensor.targets[writer_index].device_index] += 1; + } + } + // Every pump is held while the request's pieces land, and every + // queue is reserved before any changes, so the request is either + // fully queued or not queued at all. + lockAllPumps(loader); + { + errdefer unlockAllPumps(loader); + // A failure recorded before these locks may have drained the + // queues already (`abortReady`), and nothing would complete what + // this queued; one recorded after them drains it. + if (loader.first_error.get()) |err| return err; + for (loader.pumps, counts[0..loader.pumps.len]) |*device_pump, count| { + try device_pump.queue.ensureUnusedCapacity(loader.allocator, count); + } + } + // Every piece holds the request, and with it the slot, until its + // callback. + _ = request.pending.fetchAdd(pieces, .acq_rel); + for (transfers) |transfer| { + // The planner cut the transfers to the job's blocks. + const data = blockPiece(blocks, block_size, transfer.landing, transfer.len); + std.debug.assert(data.len == transfer.len); + const tensor = transfer.destination.state.readyValue().?; + var mask = transfer.writer_mask; + while (mask != 0) { + const writer_index: usize = @intCast(@ctz(mask)); + mask &= mask - 1; + const target = &tensor.targets[writer_index]; + loader.pumps[target.device_index].queue.pushBackAssumeCapacity(.{ + .target = target, + .request = request, + .data = data, + .destination_offset = transfer.destination_offset, + }); + } + } + unlockAllPumps(loader); + for (counts[0..loader.pumps.len], 0..) |count, device_index| { + if (count != 0) requestPump(loader, device_index); + } + } + + /// Runs the device's pump on this thread unless one is already running. + fn requestPump(loader: *Loader, device_index: usize) void { + const device_pump = &loader.pumps[device_index]; + device_pump.mutex.lockUncancelable(loader.io); + if (device_pump.pumping or loader.first_error.get() != null) { + device_pump.mutex.unlock(loader.io); + return; + } + device_pump.pumping = true; + device_pump.mutex.unlock(loader.io); + pump(loader, device_index); + } + + /// Submits from the device's queue, in arrival order, until the device + /// has no room or nothing is queued. Any queued piece can go: nothing + /// waits for another piece. Exactly one pump runs per device at a time, + /// see `DevicePump.pumping`. + fn pump(loader: *Loader, device_index: usize) void { + const device_pump = &loader.pumps[device_index]; + while (true) { + var selected: ?Piece = null; + device_pump.mutex.lockUncancelable(loader.io); + device_pump.destroyRetired(); + if (loader.first_error.get() == null and device_pump.hasRoom(loader.dma_budget_bytes)) { + if (device_pump.queue.popFront()) |piece| { + device_pump.active_bytes += piece.data.len; + device_pump.active_pieces += 1; + selected = piece; + } + } + if (selected == null) device_pump.pumping = false; + device_pump.mutex.unlock(loader.io); + if (selected) |piece| submitOne(loader, piece) else return; + } + } + + fn submitOne(loader: *Loader, piece: Piece) void { + const device_index = piece.target.device_index; + const len = piece.data.len; + submitTransfer(loader, piece) catch |err| { + loader.recordError(err, "submit DMA transfer: device={d}, destination_offset={d}, bytes={d}", .{ device_index, piece.destination_offset, len }); + finishPiece(loader, device_index, len, null, piece.request); + }; + } + + fn submitTransfer(loader: *Loader, piece: Piece) !void { + const api = loader.platform.pjrt_api; + const target = piece.target; + const is_last = target.nextIsLast(piece.data.len); + const event = try target.manager.transferData( + api, + 0, + piece.data, + @intCast(piece.destination_offset), + is_last, + ); + target.noteSubmitted(piece.data.len); + + // The plan holds one event slot per planned submission; the batch + // owns it. + const plan = piece.request.plan; + const event_index = plan.events_used.fetchAdd(1, .monotonic); + std.debug.assert(event_index < plan.events.len); + const ctx = &plan.events[event_index]; + ctx.* = .{ + .request = piece.request, + .pjrt_event = event, + .device_index = target.device_index, + .len = piece.data.len, + }; + + event.onReady(api, EventContext, struct { + fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { + const request = ctx_.request; + const loader_ = request.loader; + const api_ = loader_.platform.pjrt_api; + const device_index = ctx_.device_index; + const len = ctx_.len; + // The shipped plugins resolve this event without an error + // whatever happened to the copy (the C API wrapper sets the + // promise with an OK status); kept for a plugin that reports. + // The callback owns the error. + if (err) |pjrt_error| { + loader_.recordError(pjrt_error.getCode(api_).toApiError(), "complete DMA transfer: device={d}, bytes={d}, message={s}", .{ device_index, len, pjrt_error.getMessage(api_) }); + pjrt_error.deinit(api_); + } + finishPiece(loader_, device_index, len, ctx_, request); + } + }.call, ctx) catch |err| { + event.awaitRaw(api) catch {}; + ctx.destroyEvent(); + return err; + }; + } + + /// Ends a submitted piece: frees its room, which may pump on this thread; + /// then retires a fired context, after that pump, so its event is never + /// destroyed inside its own callback; then releases the request last, + /// since that release may complete and free the batch. + fn finishPiece(loader: *Loader, device_index: usize, len: usize, fired: ?*EventContext, request: *ReadRequest) void { + eventCompleted(loader, device_index, len); + if (fired) |ctx| retireEvent(loader, ctx); + request.release(); + } + + fn eventCompleted(loader: *Loader, device_index: usize, len: usize) void { + const device_pump = &loader.pumps[device_index]; + device_pump.mutex.lockUncancelable(loader.io); + std.debug.assert(device_pump.active_bytes >= len); + std.debug.assert(device_pump.active_pieces > 0); + device_pump.active_bytes -= len; + device_pump.active_pieces -= 1; + device_pump.mutex.unlock(loader.io); + // A ready callback can be the first place an asynchronous PJRT error + // becomes visible. Once outside the pump lock, retire every queued + // piece so request lifecycles cannot wait forever on entries that + // the failed pumps will no longer submit. + if (loader.first_error.get() != null) + abortReady(loader) + else + requestPump(loader, device_index); + } + + /// Force-completes every queued piece, one device at a time. A + /// completion here may finish its batch, but a batch still has an entry + /// queued until its last one is popped, and it is retired with every + /// pump locked, so no queued entry is touched after its request ends. + fn abortReady(loader: *Loader) void { + for (loader.pumps) |*device_pump| { + device_pump.mutex.lockUncancelable(loader.io); + while (device_pump.queue.popFront()) |piece| { + piece.request.release(); + } + device_pump.mutex.unlock(loader.io); + } + } +}; + const DestinationState = struct { /// One destination buffer of a tensor. PJRT makes the buffer ready once /// every transfer submitted to it has completed and one of them carried @@ -1369,6 +1719,18 @@ test "fifo scheduler concurrent claims across two batches return each job once" // DMA pump and transfer target tests. +test "device pump admits by bytes and by pieces" { + var pump: Dma.DevicePump = .{}; + try std.testing.expect(pump.hasRoom(8)); + // The piece that crosses the budget was admitted; the next is not. + pump.active_bytes = 8; + try std.testing.expect(!pump.hasRoom(8)); + pump.active_bytes = 7; + try std.testing.expect(pump.hasRoom(8)); + pump.active_pieces = max_dma_pieces_per_device; + try std.testing.expect(!pump.hasRoom(8)); +} + test "the submission that completes a target's bytes carries the last flag" { var target: DestinationState.Target = .{ .manager = undefined, .device_index = 0, .total = 100 }; try std.testing.expect(!target.fullySubmitted()); @@ -1384,6 +1746,140 @@ test "the submission that completes a target's bytes carries the last flag" { try std.testing.expect(target.fullySubmitted()); } +// Request, credit and batch lifecycle tests. + +/// A single-device loader without PJRT, workers or pinned arenas: enough +/// for request, credit and batch lifecycle tests, with 64-byte blocks cut +/// as `create` cuts them, `slots` credits of one block. `platform` and +/// `workspace` are its only undefined fields: what these tests run reaches +/// the platform only through a non-null PJRT event +/// (`Dma.EventContext.destroyEvent`). Must not move after `init`. +const TestLoader = struct { + memory: []u8, + pumps: [1]Dma.DevicePump = .{.{}}, + loader: Loader, + + fn init(self: *TestLoader, slots: usize) !void { + const allocator = std.testing.allocator; + const memory = try allocator.alloc(u8, slots * 64); + errdefer allocator.free(memory); + const blocks = try allocator.alloc(host_memory.Block, slots); + errdefer allocator.free(blocks); + for (blocks, 0..) |*block, index| block.* = memory[index * 64 ..][0..64]; + self.* = .{ .memory = memory, .loader = undefined }; + self.loader = .{ + .allocator = allocator, + .io = std.testing.io, + .platform = undefined, + .workspace = undefined, + .blocks = blocks, + .scheduler = .init(allocator), + .read_permits = .{ .permits = 1 }, + .read_width = 1, + .credits = try .init(allocator, blocks, 1), + .pumps = &self.pumps, + .dma_budget_bytes = 64, + .plan_config = .{ .block_size = 64, .request_size = 64 }, + }; + } + + fn deinit(self: *TestLoader) void { + const allocator = std.testing.allocator; + self.loader.scheduler.deinit(); + self.pumps[0].queue.deinit(allocator); + self.loader.credits.deinit(allocator); + allocator.free(self.loader.blocks); + allocator.free(self.memory); + } +}; + +test "a callback failing while a piece is queued drains it and completes the batch" { + const io = std.testing.io; + const allocator = std.testing.allocator; + var fixture: TestLoader = undefined; + try fixture.init(1); + defer fixture.deinit(); + const loader = &fixture.loader; + + // One request whose only piece is queued behind a DMA slot that a failed + // callback is about to free. + const batch = try publishTestBatch(&loader.scheduler, 1); + const request = loader.claimRequest().?; + var target: DestinationState.Target = .{ .manager = undefined, .device_index = 0, .total = 64 }; + _ = request.pending.fetchAdd(1, .acq_rel); + try fixture.pumps[0].queue.pushBack(allocator, .{ + .target = &target, + .request = request, + .data = loader.credits.slotBlocks(request.slot)[0], + .destination_offset = 0, + }); + fixture.pumps[0].active_bytes = 64; + fixture.pumps[0].active_pieces = 1; + request.release(); + try std.testing.expect(!batch.done.isSet()); + try std.testing.expectEqual(@as(usize, 1), loader.credits.inUse(io)); + _ = loader.first_error.set(error.Unknown); + + Dma.eventCompleted(loader, 0, 64); + try std.testing.expectEqual(@as(usize, 0), fixture.pumps[0].active_pieces); + try std.testing.expectEqual(@as(usize, 0), request.pending.load(.acquire)); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expect(batch.done.isSet()); + + Dma.retireBatch(loader, batch); + batch.destroy(); +} + +test "retired events are destroyed by the next pump or by a batch's retirement" { + // Without PJRT, contexts with a null event stand in for destroyed ones; + // the list mechanics are the subject. + const io = std.testing.io; + var fixture: TestLoader = undefined; + try fixture.init(2); + defer fixture.deinit(); + const loader = &fixture.loader; + + const batch = try publishTestBatch(&loader.scheduler, 2); + const requests = [_]*ReadRequest{ + loader.claimRequest().?, + loader.claimRequest().?, + }; + const plan = batch.plans.items[0]; + plan.events_used.store(2, .release); + for (plan.events, requests) |*ctx, request| ctx.* = .{ + .request = request, + .pjrt_event = null, + .device_index = 0, + .len = 64, + }; + // Two callbacks fired: the next pump destroys both, newest first. + Dma.retireEvent(loader, &plan.events[0]); + Dma.retireEvent(loader, &plan.events[1]); + const device_pump = &fixture.pumps[0]; + try std.testing.expect(device_pump.retired == &plan.events[1]); + try std.testing.expect(plan.events[1].next_retired == &plan.events[0]); + device_pump.mutex.lockUncancelable(io); + device_pump.destroyRetired(); + device_pump.mutex.unlock(io); + try std.testing.expect(device_pump.retired == null); + try std.testing.expect(plan.events[0].next_retired == null); + try std.testing.expect(plan.events[1].next_retired == null); + + // A callback that fires after the last pump leaves its context linked; + // the batch's retirement destroys it before the batch is freed, with + // the retired contexts of a batch still in flight. + var other: Dma.EventContext = .{ .request = undefined, .pjrt_event = null, .device_index = 0, .len = 64 }; + Dma.retireEvent(loader, &other); + Dma.retireEvent(loader, &plan.events[0]); + for (requests) |request| request.release(); + try std.testing.expect(batch.done.isSet()); + Dma.retireBatch(loader, batch); + try std.testing.expect(device_pump.retired == null); + try std.testing.expect(plan.events[0].next_retired == null); + try std.testing.expect(other.next_retired == null); + batch.destroy(); +} + // Source read tests. test "readFilePositionalAllV resumes short reads and stops at minimum" { From 95d0aa3025321f03ef6bca8d7f93c48870f664dc Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:13:27 +0100 Subject: [PATCH 071/103] zml/io: add the direct runtime: workers, read permits, source slots, sticky error, progress Loader is the direct backend that loads end to end: whole tensors into device memory, with exact reads. create carves the workspace into the lifecycle credits' slots and starts width + 1 workers, which share width read permits, so a full width of reads is in flight while one worker queues what it read. Each worker takes a credit before it claims a job, so a claimed job never waits for its blocks. submit creates the batch's destinations and parts in its arena, then plans and publishes one file at a time and drops the publish sentinel. A planning failure fails the loader, since a partial submission can never complete. The workers may already hold the plans it published, so the batch cannot just be destroyed: submit seals it, awaits what was published and returns the first error. awaitBatch waits for the batch, checks that a clean batch submitted every byte of every target, retires its events and destroys it with its transfer managers, whose destructor errors every buffer a failure left unfinished, so such an output never reports success. abort fails the loader; destroy stops the workers and frees everything. Each source path gets one slot holding its own copy of the path, opened by the first read and closed by destroy. Every read takes a permit and stops once the loader failed. recordError keeps the first error and logs it once, retires every unclaimed job and releases every queued piece; any thread may call it, a DMA callback included. Each destination completes one progress item when its last byte is read. The tests cover request, credit and batch lifecycles on a loader without PJRT, and whole loads on CPU: the read width in flight, a read failing mid-load, a failed load's unfinished outputs on two devices, source slots, and tensors sharded by rows or columns, replicated or joined from two files on four devices. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 759 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 751 insertions(+), 8 deletions(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 3b5e72b0..6c6a5ba0 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -1,6 +1,7 @@ const std = @import("std"); const pjrt = @import("pjrt"); +const stdx = @import("stdx"); const Buffer = @import("../buffer.zig").Buffer; const pjrtx = @import("../pjrtx.zig"); @@ -8,8 +9,13 @@ const Platform = @import("../platform.zig").Platform; const safetensors = @import("../safetensors.zig"); const Shape = @import("../shape.zig").Shape; const backend = @import("backend.zig"); +const BackendOptions = backend.Options; +const LoadPart = backend.LoadPart; +const LoadSpec = backend.LoadSpec; const ShardGrid = @import("ShardGrid.zig"); const host_memory = @import("host_memory.zig"); +const load_log = @import("log.zig").load; +const testing = @import("../testing.zig"); /// Creating a tensor's device state: host memory or PJRT, since the front /// end checked its placement. `LazyOnce` replays it to every later reader @@ -24,6 +30,327 @@ const max_dma_pieces_per_device: usize = 64; /// Each device's in-flight DMA budget, in blocks (`Dma.DevicePump.hasRoom`). const dma_depth: usize = 8; +/// The direct DMA backend. Submissions and awaits come from one task at a +/// time; the workers and the pumps run concurrently with them. +pub const Loader = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + /// The pinned arenas, grown once at creation and cut into `blocks`. + workspace: host_memory.Workspace, + /// The workspace's blocks (`carve`): the credits' slots (`Credits`). + blocks: []const host_memory.Block, + scheduler: Scheduler, + /// One permit per source read in flight (`readSlot`), `read_width` of + /// them. The width never changes: a source that answers 429 (or 503 on + /// the object stores) is held one layer down, where the VFS holds every + /// request of that backend (`vfs/request.zig`), so the traffic falls + /// while the memory bounds stay. + read_permits: std.Io.Semaphore, + read_width: usize, + /// Lifecycle credits (`Credits`). + credits: Credits, + /// The sticky first error (`recordError`). + first_error: stdx.FirstError(anyerror) = .{}, + /// One per device (`Dma.DevicePump`). Where several are locked at once + /// (`Dma.enqueue`, `Dma.retireBatch`) they are taken in device + /// order; a pump, a completion and `Dma.abortReady` hold one at a time. + pumps: []Dma.DevicePump, + /// Per-device in-flight budget: `dma_depth` blocks, in bytes. + dma_budget_bytes: usize, + worker_group: std.Io.Group = .init, + /// One per source path. Its file opens at its first read and stays + /// open until `destroy`. Keyed by the slot's own copy of the path: the + /// registry that named it may go once the batch retired. + source_slots: std.StringHashMapUnmanaged(*SourceSlot) = .empty, + /// Submissions so far; the next batch's sequence number. + batch_count: usize = 0, + /// The block size and the request size, for the whole load. + plan_config: Planner.Config, + + pub fn create( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: BackendOptions, + ) !*Loader { + // A destination's shards and a transfer's writers (a u64 mask) are + // per device; Platform refuses devices past its maximum. + comptime std.debug.assert(Platform.MAX_NUM_DEVICES <= ShardGrid.max_boxes); + const self = try allocator.create(Loader); + errdefer allocator.destroy(self); + + var workspace: host_memory.Workspace = try .init(allocator, io, platform); + errdefer workspace.deinit(); + // Every read is exact: no alignment. + const block_size = opts.dmaBlockSizes()[0]; + const request_size = backend.requestSize(opts.read_hints.read_chunk_size, block_size, 0).?; + const blocks_per_request = request_size / block_size; + const width = opts.read_parallelism; + // The workspace logs what it fails to map. + const blocks = try workspace.carve(block_size, 0, (width + 1) * blocks_per_request); + var credits: Credits = try .init(allocator, blocks, blocks_per_request); + errdefer credits.deinit(allocator); + const workers = width + 1; + std.debug.assert(credits.count() >= workers); + const pumps = try allocator.alloc(Dma.DevicePump, platform.devices.len); + errdefer allocator.free(pumps); + @memset(pumps, .{}); + + self.* = .{ + .allocator = allocator, + .io = io, + .platform = platform, + .workspace = workspace, + .blocks = blocks, + .scheduler = .init(allocator), + .read_permits = .{ .permits = width }, + .read_width = width, + .credits = credits, + .pumps = pumps, + .dma_budget_bytes = dma_depth * block_size, + .plan_config = .{ + .block_size = block_size, + .request_size = request_size, + }, + }; + std.debug.assert(self.dma_budget_bytes > 0); + errdefer self.scheduler.deinit(); + + errdefer self.stopWorkers(); + for (0..workers) |_| self.worker_group.concurrent(io, workerMain, .{self}) catch |err| { + load_log.err("start {d} source workers: {s}", .{ workers, @errorName(err) }); + return err; + }; + return self; + } + + /// Plans `specs` one source file at a time, publishes each file's plan + /// as soon as it exists behind every earlier submission, then seals the + /// batch: work on the first file starts while the rest is planned. The + /// batch owns its destinations and parts until `awaitBatch` retires it. + /// The specs keep the `LoadSpec` contract, which the front end checked, + /// and this backend's: device memory, and contiguous parts covering + /// every byte of their destination. A planning failure fails the + /// loader (a partial submission can never complete): the batch is + /// awaited here and the caller sees only the first error. + pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { + const batch = batch: { + const batch = try Batch.create(self.allocator, self.io, self.batch_count); + errdefer batch.destroy(); + try self.createDestinations(batch, specs, progress); + break :batch batch; + }; + Planner.publishFiles(&self.scheduler, self.io, batch, batch.parts, self.plan_config) catch |err| { + // Fail the loader, seal and await what was published of this + // batch, and return the loader's first error. + self.recordError(err, "plan batch {d}, published_plans={d}", .{ batch.sequence, batch.plans.items.len }); + self.batch_count += 1; + batch.releaseUnits(1); + try self.awaitBatch(batch); + unreachable; + }; + self.batch_count += 1; + // Every plan is visible: drop the publish sentinel. A batch without + // jobs completes right here. + batch.releaseUnits(1); + return batch; + } + + /// Waits for the batch's last completion unit, retires it and returns + /// the loader's sticky error if the loader failed. Destroying the batch + /// destroys its transfer managers, and PJRT's manager destructor errors + /// every buffer whose last transfer never went out, so a buffer the + /// failure left open reports that error, never success. A transfer + /// that fails asynchronously errors its buffer's definition event too + /// (the done event carries no error). A program reading such a buffer + /// fails its execution; a bulk buffer shows the error at its first use. + /// Either way an output of a failed submission holds buffers whose + /// contents are unspecified, or was never written. A batch that + /// completed without an error submitted every byte of every target (the + /// pump flagged the submission that completed its bytes); one that did + /// not would leave a buffer PJRT then errors, a bookkeeping bug this + /// panics on. + pub fn awaitBatch(self: *Loader, batch: *Batch) !void { + batch.done.waitUncancelable(self.io); + // Every request of this batch has completed: no worker or callback + // touches its managers or contexts any more. + const load_error = self.first_error.get(); + if (load_error == null) for (batch.destinations) |*destination| { + const submitted = submitted: { + const state = destination.state.readyValue() orelse break :submitted false; + for (state.targets) |target| { + if (!target.fullySubmitted()) break :submitted false; + } + break :submitted true; + }; + if (!submitted) std.debug.panic("loader: batch {d} completed without submitting every byte of tensor {s} {f}", .{ batch.sequence, destination.name(), destination.shape }); + }; + // Each destination completed its progress item. + if (load_error == null) for (batch.destinations) |*destination| std.debug.assert(destination.unread_bytes.load(.acquire) == 0); + Dma.retireBatch(self, batch); + self.destroyBatch(batch); + if (load_error) |err| return err; + } + + /// Fails the loader unless it already failed (`recordError`): unclaimed + /// jobs retire, queued pieces drain, and requests in flight finish + /// their read. + pub fn abort(self: *Loader) void { + self.recordError(error.Canceled, "loader aborted", .{}); + } + + /// The caller awaited every batch before this; nothing is queued or in + /// flight, so stopping the workers is a plain shutdown. + pub fn destroy(self: *Loader) void { + self.stopWorkers(); + var slots = self.source_slots.valueIterator(); + while (slots.next()) |slot| { + slot.*.deinit(self.io); + self.allocator.free(slot.*.uri); + self.allocator.destroy(slot.*); + } + self.source_slots.deinit(self.allocator); + std.debug.assert(self.credits.inUse(self.io) == 0); + for (self.pumps) |*device_pump| { + std.debug.assert(device_pump.active_pieces == 0); + std.debug.assert(device_pump.queue.len == 0); + std.debug.assert(device_pump.retired == null); + device_pump.queue.deinit(self.allocator); + } + self.allocator.free(self.pumps); + self.scheduler.deinit(); + self.credits.deinit(self.allocator); + self.workspace.deinit(); + self.allocator.destroy(self); + } + + /// Creates the batch's destinations, one per spec, and their parts in + /// its arena, their sources and dimensions staying the specs' until the + /// batch is awaited (`backend.LoadSpec`). A destination's device state + /// is created lazily by the first worker that reads for it. + fn createDestinations( + self: *Loader, + batch: *Batch, + specs: []const LoadSpec, + progress: std.Progress.Node, + ) !void { + const arena = batch.arena.allocator(); + var part_count: usize = 0; + for (specs) |spec| part_count += spec.parts.len; + const destinations = try arena.alloc(Destination, specs.len); + const parts = try arena.alloc(Part, part_count); + var next_part: usize = 0; + for (specs, destinations) |spec, *destination| { + std.debug.assert(spec.memory == .default); // submit + const grid: ShardGrid = .init(self.platform, spec.shape); + const destination_parts = parts[next_part..][0..spec.parts.len]; + next_part += spec.parts.len; + var covered_bytes: u64 = 0; + for (spec.parts, destination_parts) |part, *slot| { + std.debug.assert(part.lattice.dims.len == 0); // submit + slot.* = .{ + .source = part.source, + .source_slot = try self.sourceSlot(part.source.file_uri), + .destination = destination, + .lattice = part.lattice, + }; + covered_bytes += part.lattice.width; + } + // The parts are disjoint (`backend.LoadSpec`): they cover the + // destination when their widths add up to it. + std.debug.assert(covered_bytes == grid.total); // submit + destination.* = .{ + .shape = spec.shape, + .grid = grid, + .output = spec.output, + .parts = destination_parts, + .unread_bytes = .init(covered_bytes), + .progress = progress, + }; + } + batch.destinations = destinations; + batch.parts = parts; + } + + /// Releases the destinations' device state, then the batch's arena. + fn destroyBatch(self: *Loader, batch: *Batch) void { + for (batch.destinations) |*destination| destination.deinit(self.allocator, self.platform.pjrt_api); + batch.destroy(); + } + + fn sourceSlot(self: *Loader, uri: []const u8) !*SourceSlot { + if (self.source_slots.get(uri)) |slot| return slot; + const owned = try self.allocator.dupe(u8, uri); + errdefer self.allocator.free(owned); + const slot = try self.allocator.create(SourceSlot); + errdefer self.allocator.destroy(slot); + slot.* = .{ .uri = owned }; + try self.source_slots.putNoClobber(self.allocator, owned, slot); + return slot; + } + + fn workerMain(self: *Loader) void { + var scratch = ReadRequest.Scratch.init(self.allocator, self.credits.blocks_per_slot) catch |err| { + self.recordError(err, "allocate source worker scratch: blocks={d}", .{self.credits.blocks_per_slot}); + return; + }; + defer scratch.deinit(); + while (self.scheduler.waitForWork(self.io)) { + const request = self.claimRequest() orelse continue; + // The scheduling sentinel keeps the batch, and with it the + // request's plan, alive through `run` and error reporting. + defer request.release(); + request.run(&scratch) catch |err| { + self.recordError(err, "source request: file={s}, job={t}", .{ request.plan.source_slot.uri, request.plan.jobs[request.index].kind }); + }; + } + } + + /// Takes a credit, then the next job (`Credits`): a claimed job never + /// waits for its slot. Null, with the credit given back, when nothing + /// was left to claim. + fn claimRequest(self: *Loader) ?*ReadRequest { + const slot = self.credits.acquire(self.io); + const claim = self.scheduler.claim(self.io) orelse { + self.credits.release(self.io, slot); + return null; + }; + return ReadRequest.init(self, claim, slot); + } + + /// Stops the workers: every batch was awaited, so nothing is queued and + /// no request holds a credit or permit, and `Scheduler.fail` only wakes + /// the workers waiting for work. A worker may still hold the credit it + /// took for a claim that finds nothing (`claimRequest`); it gives it + /// back before it sees `stopping`, so every credit is free once this + /// returns. Awaiting a group that never spawned returns at once, so + /// this also cleans up a failed `create`. + fn stopWorkers(self: *Loader) void { + std.debug.assert(self.scheduler.queue.len == 0); + self.scheduler.fail(self.io); + self.worker_group.await(self.io) catch {}; + } + + /// Keeps `err` as the sticky error unless one is kept already and logs + /// it, then fails the load without waiting: unclaimed jobs retire + /// (`Scheduler.fail`) and queued pieces are released + /// (`Dma.abortReady`). Any thread may call it, a DMA callback included; + /// requests in flight stop at their next read (`readSlot`) or enqueue + /// and release their units. + /// Failing other batches than the caller's is safe from any thread: + /// `Scheduler.fail` and `Dma.abortReady` reach only live batches (their + /// docs), and `destroy` waits for the worker group before it frees what + /// the workers use. + fn recordError(self: *Loader, err: anyerror, comptime fmt: []const u8, args: anytype) void { + if (self.first_error.set(err)) { + load_log.failure(err, fmt, args); + self.scheduler.fail(self.io); + Dma.abortReady(self); + } + } +}; + /// One submission: its plans (one per source file, published as each file /// is planned), the destinations it writes and their parts, and every /// request and event context created while loading it, all in the batch's @@ -89,11 +416,11 @@ pub const Batch = struct { } }; -/// One output buffer and what places it. Its parts may come from several -/// files, so several plans can write it: their pieces all count toward the -/// same targets (`Target.total` is the placement's bytes, whichever plan -/// submits them), and its device state is created once, by the first -/// worker that reads for it. +/// One output buffer and what places it. Owned by the batch. Its parts may +/// come from several files, so several plans can write it: their pieces all +/// count toward the same targets (`Target.total` is the placement's bytes, +/// whichever plan submits them), and its device state is created once, by +/// the first worker that reads for it. const Destination = struct { /// The caller's shape, which the output buffer keeps. shape: Shape, @@ -104,6 +431,10 @@ const Destination = struct { /// What fills it, in ascending, disjoint destination order /// (`backend.LoadSpec`). parts: []Part, + /// Bytes not read yet, the parts' widths at first. The read that + /// brings it to zero completes the destination's item on `progress`. + unread_bytes: std.atomic.Value(u64), + progress: std.Progress.Node = .none, state: LazyOnce(DestinationState) = .{}, fn ensureState(self: *Destination, allocator: std.mem.Allocator, io: std.Io, platform: *const Platform) StateInitError!*DestinationState { @@ -113,6 +444,17 @@ const Destination = struct { fn deinit(self: *Destination, allocator: std.mem.Allocator, api: *const pjrt.Api) void { if (self.state.readyValue()) |state| state.deinit(allocator, api); } + + /// The first source's name, for logs. + fn name(self: *const Destination) []const u8 { + return self.parts[0].source.name; + } + + /// Counts `bytes` as read (`unread_bytes`); the last read completes the + /// destination's one progress item. + fn recordRead(self: *Destination, bytes: u64) void { + if (self.unread_bytes.fetchSub(bytes, .acq_rel) == bytes) self.progress.completeOne(); + } }; /// One source range of a destination: what the planner sorts by file offset @@ -133,9 +475,20 @@ const Part = struct { } }; -/// One source file: every job of a plan reads the file of its slot. +/// One source file: every job of a plan reads the file of its slot, which +/// the first read opens. const SourceSlot = struct { uri: []const u8, + file: LazyOnce(std.Io.File) = .{}, + + fn ensure(self: *SourceSlot, io: std.Io) std.Io.File.OpenError!std.Io.File { + const file = try self.file.ensure(io, std.Io.Dir.openFile, .{ .cwd(), io, self.uri, .{ .mode = .read_only } }); + return file.*; + } + + fn deinit(self: *SourceSlot, io: std.Io) void { + if (self.file.readyValue()) |file| file.close(io); + } }; /// One file's jobs in claim order with their transfer records and every @@ -772,6 +1125,24 @@ fn readFilePositionalAllV( return completed; } +/// Every worker read goes through here, under a read permit: a failure is +/// recorded with the file. Once the loader failed, returns its first error. +/// Consumes `iovecs` (`readFilePositionalAllV`). +fn readSlot(loader: *Loader, slot: *SourceSlot, iovecs: [][]u8, file_offset: u64, minimum_len: usize) !void { + const io = loader.io; + const file = slot.ensure(io) catch |err| { + loader.recordError(err, "open {s}", .{slot.uri}); + return err; + }; + loader.read_permits.waitUncancelable(io); + defer loader.read_permits.post(io); + if (loader.first_error.get()) |err| return err; + _ = readFilePositionalAllV(io, file, iovecs, file_offset, minimum_len) catch |err| { + loader.recordError(err, "read {s} at {d}, {d} bytes at least", .{ slot.uri, file_offset, minimum_len }); + return err; + }; +} + /// The bytes from offset `at` of `blocks` taken end to end, `block_size` /// bytes each, up to `len` of them or the end of that block. fn blockPiece(blocks: []const host_memory.Block, block_size: usize, at: usize, len: usize) []u8 { @@ -1328,6 +1699,15 @@ fn LazyOnce(comptime T: type) type { }; } +/// Writes `contents` to a new file `name` in `dir` and returns its real +/// path, in `path_buffer`. +fn writeTestFile(dir: std.Io.Dir, io: std.Io, name: []const u8, contents: []const u8, path_buffer: []u8) ![]const u8 { + const file = try dir.createFile(io, name, .{ .read = true }); + defer file.close(io); + try file.writePositionalAll(io, contents, 0); + return path_buffer[0..try file.realPath(io, path_buffer)]; +} + // Planner tests. /// `Planner.preparePlan`, kept in `arena`; the testing allocator checks the @@ -1336,8 +1716,8 @@ fn prepareTestPlan(arena: std.mem.Allocator, parts: []Part, order: []const usize return Planner.preparePlan(std.testing.allocator, arena, parts, order, config); } -/// One plain destination per source, as whole-tensor specs make them: for -/// planner tests. +/// One plain destination per source, as `createDestinations` builds them +/// for whole-tensor specs, without device state: for planner tests. fn initPlainParts( platform: *const Platform, sources: []safetensors.Tensor, @@ -1358,6 +1738,7 @@ fn initPlainParts( .grid = .init(platform, source.shape), .output = output, .parts = @as(*[1]Part, part), + .unread_bytes = .init(source.byteSize()), }; } } @@ -1830,6 +2211,65 @@ test "a callback failing while a piece is queued drains it and completes the bat batch.destroy(); } +test "batch completes when every claimed request completes" { + const io = std.testing.io; + var fixture: TestLoader = undefined; + try fixture.init(3); + defer fixture.deinit(); + const loader = &fixture.loader; + + const batch = try publishTestBatch(&loader.scheduler, 3); + try std.testing.expectEqual(@as(usize, 3), batch.remaining.load(.acquire)); + var requests: [3]*ReadRequest = undefined; + for (&requests) |*request| request.* = loader.claimRequest().?; + try std.testing.expect(loader.scheduler.claim(io) == null); + try std.testing.expectEqual(@as(usize, 3), loader.credits.inUse(io)); + + for (requests, 0..) |request, index| { + try std.testing.expect(!batch.done.isSet()); + try std.testing.expectEqual(requests.len - index, batch.remaining.load(.acquire)); + request.release(); + } + try std.testing.expect(batch.done.isSet()); + try std.testing.expectEqual(@as(usize, 0), batch.remaining.load(.acquire)); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + // Each claim took its job's slot of the plan's request array. + for (requests, batch.plans.items[0].requests) |request, *slot| { + try std.testing.expect(request == slot); + } + + Dma.retireBatch(loader, batch); + batch.destroy(); +} + +test "a worker waiting for a credit when the loader fails gets one back and stops" { + const io = std.testing.io; + var fixture: TestLoader = undefined; + try fixture.init(1); + defer fixture.deinit(); + const loader = &fixture.loader; + + // A request in flight holds the only credit; a worker that saw the + // second job waits on the credits. + const batch = try publishTestBatch(&loader.scheduler, 2); + const request = loader.claimRequest().?; + try loader.worker_group.concurrent(io, Loader.workerMain, .{loader}); + while (loader.credits.condition.state.load(.monotonic).waiters == 0) io.sleep(.fromMilliseconds(1), .awake) catch {}; + + // The failure retires the second job and closes nothing: the request + // gives its credit back as it ends, the worker claims nothing with it, + // and `destroy`'s shutdown stops it. + loader.recordError(error.InputOutput, "injected source failure", .{}); + try std.testing.expect(!batch.done.isSet()); + request.release(); + loader.stopWorkers(); + try std.testing.expect(batch.done.isSet()); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expectEqual(@as(usize, 0), batch.plans.items[0].requests[1].pending.load(.acquire)); + Dma.retireBatch(loader, batch); + batch.destroy(); +} + test "retired events are destroyed by the next pump or by a batch's retirement" { // Without PJRT, contexts with a null event stand in for destroyed ones; // the list mechanics are the subject. @@ -1880,6 +2320,52 @@ test "retired events are destroyed by the next pump or by a batch's retirement" batch.destroy(); } +test "overlapping batches complete under concurrent claims and retirement" { + const io = std.testing.io; + const worker_count = 4; + var fixture: TestLoader = undefined; + try fixture.init(worker_count); + defer fixture.deinit(); + const loader = &fixture.loader; + + const Worker = struct { + fn run(loader_: *Loader) void { + while (loader_.scheduler.waitForWork(loader_.io)) if (loader_.claimRequest()) |request| request.release(); + } + }; + var group: std.Io.Group = .init; + defer { + loader.scheduler.fail(io); + group.await(io) catch {}; + } + for (0..worker_count) |_| try group.concurrent(io, Worker.run, .{loader}); + + // Three batches in flight at a time, awaited newest first: a batch is + // retired while later ones are still being claimed. + for (0..70) |round| { + var batches: [3]*Batch = undefined; + var job_counts: [3]usize = undefined; + for (&batches, &job_counts, 0..) |*batch, *job_count, offset| { + job_count.* = (round * 3 + offset) % 7; + batch.* = try publishTestBatch(&loader.scheduler, job_count.*); + } + var index = batches.len; + while (index > 0) { + index -= 1; + const batch = batches[index]; + batch.done.waitUncancelable(io); + Dma.retireBatch(loader, batch); + batch.destroy(); + } + try std.testing.expectEqual(@as(usize, 0), unclaimedJobs(&loader.scheduler)); + } + // Every request released its credit before its batch completed; the + // workers only hold one while a claim is in progress. + loader.scheduler.fail(io); + try group.await(io); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); +} + // Source read tests. test "readFilePositionalAllV resumes short reads and stops at minimum" { @@ -1931,3 +2417,260 @@ test "lazy initialization preserves its finite error across repeated callers" { try std.testing.expectError(error.FileNotFound, once.ensure(std.testing.io, Context.init, .{&calls})); try std.testing.expectEqual(1, calls); } + +// Whole-loader tests. + +test "loader releases the pinned set when its workers cannot start" { + const allocator = std.testing.allocator; + const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, std.testing.io); + // The testing io, but no worker starts: `create` fails after the carve, + // the credits and the pumps, and unwinds every one. + var vtable = std.testing.io.vtable.*; + vtable.groupConcurrent = std.Io.failingGroupConcurrent; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + const result: anyerror!void = if (Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off })) |loader| unexpected: { + loader.destroy(); + break :unexpected {}; + } else |err| err; + try std.testing.expectError(error.ConcurrencyUnavailable, result); +} + +test "exactly the width of reads are in flight, and a read failing mid-load leaves nothing held" { + const allocator = std.testing.allocator; + const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, std.testing.io); + // Every source read holds for a moment and is counted; the one numbered + // `fail_at` fails. + const Reads = struct { + var in_flight: std.atomic.Value(usize) = .init(0); + var peak: std.atomic.Value(usize) = .init(0); + var started: std.atomic.Value(usize) = .init(0); + var fail_at: usize = std.math.maxInt(usize); + + fn readPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + _ = peak.fetchMax(in_flight.fetchAdd(1, .acq_rel) + 1, .acq_rel); + defer _ = in_flight.fetchSub(1, .acq_rel); + std.testing.io.sleep(.fromMilliseconds(5), .awake) catch {}; + if (started.fetchAdd(1, .acq_rel) == fail_at) return error.InputOutput; + return std.testing.io.vtable.fileReadPositional(userdata, file, data, offset); + } + }; + var vtable = std.testing.io.vtable.*; + vtable.fileReadPositional = Reads.readPositional; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + + // Sixteen tensors with gaps between them: a read, and a job, each. + const tensors = 16; + var contents: [tensors * 8]u8 = undefined; + for (&contents, 0..) |*byte, index| byte.* = @truncate(index); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, std.testing.io, "weights.bin", &contents, &path_buffer); + var sources: [tensors]safetensors.Tensor = undefined; + var parts: [tensors]LoadPart = undefined; + var specs: [tensors]LoadSpec = undefined; + var outputs: [tensors]Buffer = undefined; + for (&sources, &parts, &specs, &outputs, 0..) |*source, *part, *spec, *output, index| { + source.* = .{ .file_uri = path, .name = "t", .shape = .init(.{4}, .u8), .offset = index * 8 }; + part.* = .whole(source); + spec.* = .{ .shape = source.shape, .output = output, .parts = part[0..1] }; + } + + for ([_]?usize{ null, 5 }) |fail_at| { + for (&outputs) |*output| output.* = .{ ._platform = platform, ._shape = sources[0].shape, ._shards = .empty }; + defer for (&outputs) |*output| output.deinit(); + // `destroy` returns: no worker waits for a credit or a permit. + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + try std.testing.expectEqual(@as(usize, 2), loader.read_width); + Reads.in_flight = .init(0); + Reads.peak = .init(0); + Reads.started = .init(0); + Reads.fail_at = fail_at orelse std.math.maxInt(usize); + const batch = try loader.submit(&specs, .none); + if (fail_at == null) { + try loader.awaitBatch(batch); + // Three workers, two permits. + try std.testing.expectEqual(@as(usize, tensors), Reads.started.load(.acquire)); + try std.testing.expectEqual(loader.read_width, Reads.peak.load(.acquire)); + try testing.expectShards(allocator, std.testing.io, &outputs[tensors - 1], contents[(tensors - 1) * 8 ..][0..4], .default); + } else { + try std.testing.expectError(error.InputOutput, loader.awaitBatch(batch)); + // The reads stopped at the failure. + try std.testing.expect(Reads.started.load(.acquire) < tensors); + } + try std.testing.expectEqual(@as(usize, 0), loader.scheduler.queue.len); + // A worker the batch woke may still hold the credit it took for a + // claim that finds nothing: every credit is free once the workers + // stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expectEqual(loader.read_width, loader.read_permits.permits); + } +} + +test "a default-memory output a failed load left unfinished reports an error on every device" { + const allocator = std.testing.allocator; + // On the accelerator when it has two devices: each plugin's transfer + // manager errors the buffers it leaves unfinished. + var devices: testing.Devices = undefined; + try devices.init(allocator, std.testing.io, 2); + defer devices.deinit(); + const platform = devices.platform; + // The read reaching the last request fails. + const Reads = struct { + var fail_from: u64 = std.math.maxInt(u64); + + fn readPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + if (offset >= fail_from) return error.InputOutput; + return std.testing.io.vtable.fileReadPositional(userdata, file, data, offset); + } + }; + var vtable = std.testing.io.vtable.*; + vtable.fileReadPositional = Reads.readPositional; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + + // A replicated tensor of three requests, a job each: no device's last + // piece goes out. + const request_size = loader.plan_config.request_size; + const contents = try allocator.alloc(u8, 3 * request_size); + defer allocator.free(contents); + testing.pattern(contents, 1); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, std.testing.io, "weights.bin", contents, &path_buffer); + var source: safetensors.Tensor = .{ .file_uri = path, .name = "t", .shape = .init(.{contents.len}, .u8), .offset = 0 }; + var part: LoadPart = .whole(&source); + var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; + defer output.deinit(); + const spec: LoadSpec = .{ .shape = source.shape, .output = &output, .parts = (&part)[0..1] }; + + Reads.fail_from = 2 * request_size; + defer Reads.fail_from = std.math.maxInt(u64); + const batch = try loader.submit((&spec)[0..1], .none); + try std.testing.expectError(error.InputOutput, loader.awaitBatch(batch)); + // The failing job created the buffers before its read. Each one is + // ready, with an error, once the batch is retired: a success would + // hand out a buffer missing its last request, a pending one would hang + // its first user. + const placed = platform.placedShape(source.shape); + try std.testing.expectEqual(placed._sharding.mesh.?.devicesInCanonicalOrder().len, output._shards.len); + for (output._shards.constSlice()) |shard| { + const event = shard.readyEvent(platform.pjrt_api); + defer event.deinit(platform.pjrt_api); + try std.testing.expect(event.isReady(platform.pjrt_api)); + const err = event.getEventError(platform.pjrt_api) orelse return error.TestUnexpectedResult; + err.deinit(platform.pjrt_api); + } +} + +test "whole tensors load on four devices, sharded by rows or columns, replicated, or joined from two files" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 4); + defer devices.deinit(); + const mesh = try devices.mesh("tp"); + + // Rows of 32 bytes, 16 per device; rows cut into four pieces of 8 + // bytes, one per device; a table on every device; and a tensor whose + // rows continue in a second file. + var rows: [64 * 32]u8 = undefined; + var cols: [1024 * 32]u8 = undefined; + var table: [3000 * 4]u8 = undefined; + var tail: [64 * 32]u8 = undefined; + testing.pattern(&rows, 1); + testing.pattern(&cols, 2); + testing.pattern(&table, 3); + testing.pattern(&tail, 4); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "first.safetensors", ®istry, &.{ + .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &rows }, + .{ .name = "cols", .dtype = "U16", .dims = &.{ 1024, 16 }, .bytes = &cols }, + .{ .name = "table", .dtype = "U8", .dims = &.{ 3000, 4 }, .bytes = &table }, + }); + try testing.writeSafetensors(allocator, io, tmp.dir, "second.safetensors", ®istry, &.{ + .{ .name = "tail", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &tail }, + }); + var joined: [rows.len + tail.len]u8 = undefined; + @memcpy(joined[0..rows.len], &rows); + @memcpy(joined[rows.len..], &tail); + const parts = [_]LoadPart{ + .whole(registry.tensors.getPtr("rows").?), + .whole(registry.tensors.getPtr("cols").?), + .whole(registry.tensors.getPtr("table").?), + // rows again, then tail after it: the first file's run overlaps. + .whole(registry.tensors.getPtr("rows").?), + .{ .source = registry.tensors.getPtr("tail").?, .lattice = .{ .dst_offset = rows.len, .width = tail.len } }, + }; + var outputs: [4]Buffer = undefined; + const specs = [_]LoadSpec{ + .{ .shape = Shape.init(.{ .row = 64, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), .output = &outputs[0], .parts = parts[0..1] }, + .{ .shape = Shape.init(.{ .row = 1024, .col = 16 }, .u16).withPartitioning(&mesh, .{ .col = .model }), .output = &outputs[1], .parts = parts[1..2] }, + .{ .shape = Shape.init(.{ .row = 3000, .col = 4 }, .u8).withPartitioning(&mesh, .{}), .output = &outputs[2], .parts = parts[2..3] }, + .{ .shape = Shape.init(.{ .row = 128, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), .output = &outputs[3], .parts = parts[3..5] }, + }; + const globals = [_][]const u8{ &rows, &cols, &table, &joined }; + + // Blocks of 4 KiB and requests of two: the first file's run is cut into + // jobs, and the jobs' transfers at block ends. + const loader = try Loader.create(allocator, io, devices.platform, .{ + .read_parallelism = 2, + .read_hints = .{ .read_chunk_size = 8192 }, + .dma_block_size = 4096, + .direct_io = .off, + }); + defer loader.destroy(); + const batch = try loader.submit(&specs, .none); + var jobs: usize = 0; + for (batch.plans.items) |plan| jobs += plan.jobs.len; + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + for (&outputs, globals) |*output, global| try testing.expectShards(allocator, io, output, global, .default); + // 46,816 bytes in the first file, 2,048 in the second. + try std.testing.expectEqual(@as(usize, 6 + 1), jobs); + // Every credit is free once the workers stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); +} + +test "source slots keep their own copy of each path" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents = [_]u8{ 1, 2, 3, 4 }; + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "weights.bin", &contents, &path_buffer); + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + // Each submission names the file through a string of its own, freed + // right after the submit: workers open the file through its slot. + for (0..2) |_| { + const uri = try allocator.dupe(u8, path); + var source: safetensors.Tensor = .{ .file_uri = uri, .name = "x", .shape = .init(.{4}, .u8), .offset = 0 }; + var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; + defer output.deinit(); + const parts = [_]LoadPart{.whole(&source)}; + const submitted = loader.submit(&.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, .none); + allocator.free(uri); + try loader.awaitBatch(try submitted); + try testing.expectShards(allocator, io, &output, &contents, .default); + } + // The testing allocator overwrites freed bytes: a slot keyed by the + // first string would miss the second lookup. + try std.testing.expectEqual(@as(usize, 1), loader.source_slots.count()); +} From b970cfd73ef5bdd22f5be2a1e79e265846a43e1b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:17:52 +0100 Subject: [PATCH 072/103] zml/io: size the direct backend with a fixed block: reserve, width, carve, zero block create now sizes the pinned set for the whole load before the first read. The DMA block is the options' one, 16 MiB unless they pin another, and a source request is the read chunk rounded up to whole blocks. Beside the workers' requests the workspace keeps a DMA reserve, eight blocks per device and at least one maximal request, which become lifecycle credits beyond the workers' own, so transfers still holding blocks keep the DMA engines fed: one credit beyond the read width moved 24.3 GiB/s on a GB300 loading DeepSeek-V4, against 43.8 GiB/s with eight requests queued. The read width is clipped to what the 16 GiB mapped ceiling leaves beside the reserve; when not even one request fits, create logs the sizes and fails with OutOfMemory. The workspace maps the reserve as one arena first, then the rest, since ROCm balances host memory per allocation, and cuts everything into blocks once, so nothing maps during a load. The first block is a block of zeros that no credit owns. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 75 +++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 12 deletions(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 6c6a5ba0..c79019fb 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -38,7 +38,9 @@ pub const Loader = struct { platform: *const Platform, /// The pinned arenas, grown once at creation and cut into `blocks`. workspace: host_memory.Workspace, - /// The workspace's blocks (`carve`): the credits' slots (`Credits`). + /// The workspace's blocks (`carve`): the zero block first, then the + /// credits' slots (`Credits`). The zero block is in no credit's slot + /// and is never written after `create`. blocks: []const host_memory.Block, scheduler: Scheduler, /// One permit per source read in flight (`readSlot`), `read_width` of @@ -48,7 +50,12 @@ pub const Loader = struct { /// while the memory bounds stay. read_permits: std.Io.Semaphore, read_width: usize, - /// Lifecycle credits (`Credits`). + /// Lifecycle credits (`Credits`), beyond the workers' through the DMA + /// reserve. One credit beyond the width fed the DMA engines one request + /// at a time: on a GB300 loading DeepSeek-V4 (pieces of 4 MiB and 256 + /// KiB) that was 24.3 GiB/s at width 16 against 43.8 GiB/s with eight + /// requests queued, and the workers spent 9 ms per request waiting for a + /// credit against 3 ms reading. credits: Credits, /// The sticky first error (`recordError`). first_error: stdx.FirstError(anyerror) = .{}, @@ -82,14 +89,47 @@ pub const Loader = struct { var workspace: host_memory.Workspace = try .init(allocator, io, platform); errdefer workspace.deinit(); - // Every read is exact: no alignment. + // The options' block for the whole load; every read is exact. const block_size = opts.dmaBlockSizes()[0]; const request_size = backend.requestSize(opts.read_hints.read_chunk_size, block_size, 0).?; const blocks_per_request = request_size / block_size; - const width = opts.read_parallelism; + // The DMA stage of every device, kept mapped beside the source + // working set; at least one maximal source request, so a single + // request always fits beside it. + const dma_reserve = @max( + dma_depth * platform.devices.len, + Planner.blockCount(backend.max_read_request_size, block_size), + ); + // One more block for the zeros, which no credit owns. + const zero_blocks = 1; + const available = workspace.capacity(block_size); + const width = @min( + opts.read_parallelism, + (available -| dma_reserve -| zero_blocks) / blocks_per_request -| 1, + ); + if (width == 0) { + load_log.err("host memory ceiling {Bi:.2} fits no source request beside the DMA reserve: available_blocks={d}, reserve_blocks={d}, blocks_per_request={d}: OutOfMemory", .{ host_memory.Workspace.mapped_bytes_ceiling, available, dma_reserve, blocks_per_request }); + return error.OutOfMemory; + } + if (width < opts.read_parallelism) { + load_log.debug("DMA source working set clipped by the mapped ceiling: width={d} of {d}", .{ + width, + opts.read_parallelism, + }); + } + // Two arenas, the reserve first: ROCm balances bytes across its host + // nodes per allocation, and that split is recorded. + const pregrowth_started: std.Io.Timestamp = .now(io, .awake); // The workspace logs what it fails to map. - const blocks = try workspace.carve(block_size, 0, (width + 1) * blocks_per_request); - var credits: Credits = try .init(allocator, blocks, blocks_per_request); + const blocks = try workspace.carve(block_size, dma_reserve, (width + 1) * blocks_per_request + dma_reserve + zero_blocks); + const pregrowth_ns: u64 = @intCast(@max(pregrowth_started.untilNow(io, .awake).nanoseconds, 0)); + load_log.debug("host workspace pregrown: mapped={Bi:.2}, pregrowth_ms={d:.3}", .{ + workspace.mappedBytes(), + @as(f64, @floatFromInt(pregrowth_ns)) / std.time.ns_per_ms, + }); + std.debug.assert(blocks.len != 0); + @memset(blocks[0], 0); + var credits: Credits = try .init(allocator, blocks[1..], blocks_per_request); errdefer credits.deinit(allocator); const workers = width + 1; std.debug.assert(credits.count() >= workers); @@ -122,6 +162,16 @@ pub const Loader = struct { load_log.err("start {d} source workers: {s}", .{ workers, @errorName(err) }); return err; }; + load_log.debug("live loader ready: target={s}, request_size={Bi:.2}, dma_block_size={Bi:.2}, dma_budget_per_device={Bi:.2}, source_width={d}, lifecycle_credits={d}, workers={d}, retained={Bi:.2}", .{ + @tagName(platform.target), + request_size, + block_size, + self.dma_budget_bytes, + width, + self.credits.count(), + workers, + self.workspace.mappedBytes(), + }); return self; } @@ -2131,9 +2181,9 @@ test "the submission that completes a target's bytes carries the last flag" { /// A single-device loader without PJRT, workers or pinned arenas: enough /// for request, credit and batch lifecycle tests, with 64-byte blocks cut -/// as `create` cuts them, `slots` credits of one block. `platform` and -/// `workspace` are its only undefined fields: what these tests run reaches -/// the platform only through a non-null PJRT event +/// as `create` cuts them, the zero block then `slots` credits of one block. +/// `platform` and `workspace` are its only undefined fields: what these +/// tests run reaches the platform only through a non-null PJRT event /// (`Dma.EventContext.destroyEvent`). Must not move after `init`. const TestLoader = struct { memory: []u8, @@ -2142,11 +2192,12 @@ const TestLoader = struct { fn init(self: *TestLoader, slots: usize) !void { const allocator = std.testing.allocator; - const memory = try allocator.alloc(u8, slots * 64); + const memory = try allocator.alloc(u8, (slots + 1) * 64); errdefer allocator.free(memory); - const blocks = try allocator.alloc(host_memory.Block, slots); + const blocks = try allocator.alloc(host_memory.Block, slots + 1); errdefer allocator.free(blocks); for (blocks, 0..) |*block, index| block.* = memory[index * 64 ..][0..64]; + @memset(blocks[0], 0); self.* = .{ .memory = memory, .loader = undefined }; self.loader = .{ .allocator = allocator, @@ -2157,7 +2208,7 @@ const TestLoader = struct { .scheduler = .init(allocator), .read_permits = .{ .permits = 1 }, .read_width = 1, - .credits = try .init(allocator, blocks, 1), + .credits = try .init(allocator, blocks[1..], 1), .pumps = &self.pumps, .dma_budget_bytes = 64, .plan_config = .{ .block_size = 64, .request_size = 64 }, From 53e8bd9f33d4bbedce9a6dd4104277a4eb3877aa Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:19:59 +0100 Subject: [PATCH 073/103] zml/io: document sizing and planning in direct_loader's header The file header describes the direct backend as the code now has it: how create sizes the block, the request, the read width, the DMA reserve and the pinned set once; how a submission is planned per file, claimed in order, read into a credit's blocks and handed to per-device pumps, with the last-transfer flag set by bytes; how a destination's bytes are placed on its devices; and the order in which the runtime's locks nest. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index c79019fb..f3244b17 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -1,3 +1,48 @@ +//! Direct loading follows a batch from per-file planning through FIFO claims, +//! source reads and DMA completion. Runtime ownership and synchronization +//! live here. +//! +//! Sizing, fixed by `Loader.create`. The DMA block is the options' one +//! (`backend.Options.dmaBlockSizes`); each device's DMA budget is +//! `dma_depth` blocks. A source request is the checkpoint's minimum read +//! (`VFS.readHints`) rounded up to whole blocks, one block at least +//! (`backend.requestSize`). The source width is `read_parallelism`, +//! clipped to what the mapped ceiling fits. The workspace then maps, once +//! and before the first read, `width + 1` requests, the DMA reserve and the +//! zero block, and cuts them into blocks (`Workspace.carve`): nothing maps +//! during a load. `width + 1` workers share `width` read permits, so a full +//! width of reads is in flight while one worker queues what it read. The +//! lifecycle credits are the whole pinned set in requests, so transfers +//! that still hold blocks keep the DMA stage fed and never want a block the +//! reads did not leave free. +//! +//! Planning and claims. `submit` sorts a submission's parts by file and +//! offset, plans one file at a time and publishes each plan at once, so the +//! workers read the first file while the next is planned. A plan merges +//! touching or overlapping ranges into reads, never across a gap or a file, +//! and cuts them into jobs of at most a request; jobs keep that file order, +//! and the `Scheduler` hands them out in submission order. A worker takes a +//! credit, claims a job, reads it into its slot's pinned blocks and queues +//! its preplanned transfer pieces on per-device pumps. A block may feed +//! several outputs or devices; the request keeps its credit until every +//! piece it queued completed or was abandoned. The last-transfer flag goes +//! on the piece that completes a destination's bytes, so reads and DMA +//! complete in any order. A batch owns its plans and callback contexts +//! until every job completes; a failure retires the unclaimed jobs and +//! drains what is in flight before the batch goes. +//! +//! Destinations. A part's source bytes map to its destination's logical +//! offsets, which the destination's `ShardGrid` places on the devices by +//! arithmetic. Pieces of one destination merge across parts when block, +//! writers and both offsets are contiguous, and its parts may come from +//! different files and plans. +//! +//! Locks. Where several pumps are held, they are taken in device order. A +//! pump's mutex may be held while a released request gives its credit back +//! (`Credits`), never the reverse, and no other lock is taken under the +//! scheduler's mutex. A DMA callback, on a plugin thread, holds one pump at +//! a time. + const std = @import("std"); const pjrt = @import("pjrt"); From b65a808cd36b07a7a518719d6d8d5420960d8b37 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:20:58 +0100 Subject: [PATCH 074/103] zml/io: refuse submits to a failed direct loader and test planning failures submit already fails the direct loader when planning a submission fails, and awaits what it published. A failed loader now also refuses a submit with its first error before it builds a batch: every later submission sees the failure that stopped the loader, and none creates destinations or plans it cannot run. The tests drive the failure paths a submission can take on CPU. With the planner's allocations failing one after another, planning fails before any plan is published, then after the first file's plan was, and past the last the read of a truncated source fails. Each failure fails the loader, leaves nothing queued and, once the workers stopped, every credit and read permit free, counts the batch once, and makes later submits return the same error. A loader that fails between a submit's open check and its first publish, as a worker of an older batch can make it, returns that failure rather than the scheduler's Canceled. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 146 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 1 deletion(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index f3244b17..410d2490 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -58,6 +58,7 @@ const BackendOptions = backend.Options; const LoadPart = backend.LoadPart; const LoadSpec = backend.LoadSpec; const ShardGrid = @import("ShardGrid.zig"); +const Memory = @import("TensorStore.zig").Memory; const host_memory = @import("host_memory.zig"); const load_log = @import("log.zig").load; const testing = @import("../testing.zig"); @@ -228,8 +229,10 @@ pub const Loader = struct { /// and this backend's: device memory, and contiguous parts covering /// every byte of their destination. A planning failure fails the /// loader (a partial submission can never complete): the batch is - /// awaited here and the caller sees only the first error. + /// awaited here and the caller sees only the first error. A failed + /// loader refuses every submit with its first error. pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { + try self.checkOpen(); const batch = batch: { const batch = try Batch.create(self.allocator, self.io, self.batch_count); errdefer batch.destroy(); @@ -320,6 +323,10 @@ pub const Loader = struct { self.allocator.destroy(self); } + fn checkOpen(self: *Loader) !void { + if (self.first_error.get()) |err| return err; + } + /// Creates the batch's destinations, one per spec, and their parts in /// its arena, their sources and dimensions staying the specs' until the /// batch is awaited (`backend.LoadSpec`). A destination's device state @@ -2516,6 +2523,14 @@ test "lazy initialization preserves its finite error across repeated callers" { // Whole-loader tests. +/// Submits `specs`, which must fail with `expected`; a batch that went out +/// anyway is awaited before the test fails. +fn expectSubmitError(loader: *Loader, specs: []const LoadSpec, expected: anyerror) !void { + const batch = loader.submit(specs, .none) catch |err| return std.testing.expectEqual(expected, err); + loader.awaitBatch(batch) catch {}; + return error.TestUnexpectedResult; +} + test "loader releases the pinned set when its workers cannot start" { const allocator = std.testing.allocator; const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch @@ -2533,6 +2548,71 @@ test "loader releases the pinned set when its workers cannot start" { try std.testing.expectError(error.ConcurrencyUnavailable, result); } +test "loader failures clean up before publication, after publication and during reading" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents = [_]u8{ 1, 2, 3, 4 }; + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "weights.bin", &contents, &path_buffer); + var later_buffer: [1024]u8 = undefined; + const later_path = try writeTestFile(tmp.dir, io, "weights.bin.later", &contents, &later_buffer); + const sources = [_]safetensors.Tensor{ + .{ .file_uri = path, .name = "first", .shape = .init(.{4}, .u8), .offset = 0 }, + .{ .file_uri = later_path, .name = "later", .shape = .init(.{5}, .u8), .offset = 0 }, + }; + const parts = [_]LoadPart{ .whole(&sources[0]), .whole(&sources[1]) }; + + // Device memory, the one kind this backend loads, goes through blocks + // and DMA. + for ([_]Memory{.default}) |memory| { + // Planning allocates from the scheduler's allocator, whose + // allocations fail in turn: the first file's planning fails before + // any plan is published, then the later file's after the first + // file's plan was. Past the last, planning succeeds and the read + // fails: the later source ends after four of five bytes. + var fail_index: usize = 0; + while (true) : (fail_index += 1) { + var outputs: [2]Buffer = @splat(.{ + ._platform = platform, + ._shape = sources[0].shape, + ._shards = .empty, + }); + defer for (&outputs) |*output| output.deinit(); + var failing: std.testing.FailingAllocator = .init(allocator, .{ .fail_index = fail_index }); + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + loader.scheduler.allocator = failing.allocator(); + const specs = [_]LoadSpec{ + .{ .shape = sources[0].shape, .memory = memory, .output = &outputs[0], .parts = parts[0..1] }, + .{ .shape = sources[1].shape, .memory = memory, .output = &outputs[1], .parts = parts[1..2] }, + }; + + if (loader.submit(&specs, .none)) |batch| { + try std.testing.expectError(error.UnexpectedEndOfFile, loader.awaitBatch(batch)); + } else |err| try std.testing.expectEqual(error.OutOfMemory, err); + try std.testing.expectEqual(@as(usize, 0), loader.scheduler.queue.len); + // A worker the batch woke may still hold the credit it took for + // a claim that finds nothing: every credit is free once the + // workers stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expectEqual(loader.read_width, loader.read_permits.permits); + + // Each failure fails the loader: a later submission gets it. + try std.testing.expectEqual(@as(usize, 1), loader.batch_count); + const expected: anyerror = if (failing.has_induced_failure) error.OutOfMemory else error.UnexpectedEndOfFile; + try std.testing.expectError(expected, loader.checkOpen()); + try expectSubmitError(loader, specs[0..1], expected); + if (!failing.has_induced_failure) break; + } + } +} + test "exactly the width of reads are in flight, and a read failing mid-load leaves nothing held" { const allocator = std.testing.allocator; const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch @@ -2740,6 +2820,70 @@ test "whole tensors load on four devices, sharded by rows or columns, replicated try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); } +test "a submit the loader fails between its open check and its first publish returns that failure" { + const io = std.testing.io; + const platform = Platform.init(std.testing.allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(std.testing.allocator, io); + // Fails the loader in the submitting task's next allocation: after + // `submit`'s open check and before its first publish, where a worker of + // an older batch can fail it and the scheduler then answers `Canceled`. + const FailNextAllocation = struct { + parent: std.mem.Allocator, + loader: *Loader = undefined, + thread: std.Thread.Id = undefined, + armed: std.atomic.Value(bool) = .init(false), + + fn arm(self: *@This(), loader: *Loader) void { + self.loader = loader; + self.thread = std.Thread.getCurrentId(); + self.armed.store(true, .release); + } + + fn allocator(self: *@This()) std.mem.Allocator { + return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + const self: *@This() = @ptrCast(@alignCast(ctx)); + // The workers allocate on threads of their own. + if (self.armed.load(.acquire) and self.thread == std.Thread.getCurrentId()) { + self.armed.store(false, .monotonic); + self.loader.recordError(error.UnexpectedEndOfFile, "an older batch's read", .{}); + } + return self.parent.rawAlloc(len, alignment, ret_addr); + } + + fn resize(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { + const self: *@This() = @ptrCast(@alignCast(ctx)); + return self.parent.rawResize(memory, alignment, new_len, ret_addr); + } + + fn remap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + const self: *@This() = @ptrCast(@alignCast(ctx)); + return self.parent.rawRemap(memory, alignment, new_len, ret_addr); + } + + fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { + const self: *@This() = @ptrCast(@alignCast(ctx)); + self.parent.rawFree(memory, alignment, ret_addr); + } + }; + var failing: FailNextAllocation = .{ .parent = std.testing.allocator }; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "weights.bin", &.{ 1, 2, 3, 4 }, &path_buffer); + var source: safetensors.Tensor = .{ .file_uri = path, .name = "x", .shape = .init(.{4}, .u8), .offset = 0 }; + var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; + defer output.deinit(); + const loader = try Loader.create(failing.allocator(), io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + const parts = [_]LoadPart{.whole(&source)}; + failing.arm(loader); + try expectSubmitError(loader, &.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, error.UnexpectedEndOfFile); +} + test "source slots keep their own copy of each path" { const allocator = std.testing.allocator; const io = std.testing.io; From a46005641aa596b9f6f2c8034b9bd72c0da3d312 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:40:25 +0100 Subject: [PATCH 075/103] zml/io: route CUDA, ROCm, oneAPI, CPU and Furiosa loads to the direct backend The Backend and Submission unions gain the direct arm, and backend.loadsDirect decides per target: CUDA, ROCm, oneAPI, CPU and Furiosa load through the direct backend; TPU, neuron and Metal stay on the buffered one until their transfer path is checked on hardware. ROCm and Furiosa stage in PJRT pinned_host arenas. Furiosa has never run this backend. supportsMemory still accepts device memory only. CPU goes direct as well, so the front end's tests on CPU now exercise the direct backend. The direct backend writes only the bytes its parts cover, so on a platform that loads through it the front end refuses a padded source with InvalidBinding instead of handing it a destination it would leave partly unwritten. A new test drives the front end over the buffered backend on CPU, so both arms stay covered. Co-Authored-By: Claude Opus 5.5 --- zml/io/backend.zig | 31 +++++++++-- zml/io/direct_loader.zig | 3 +- zml/io/host_memory.zig | 5 +- zml/io/loader.zig | 116 ++++++++++++++++++++++++++++++++------- 4 files changed, 126 insertions(+), 29 deletions(-) diff --git a/zml/io/backend.zig b/zml/io/backend.zig index 3aec6782..6f6a3502 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -1,7 +1,7 @@ //! The seam between the front end (loader.zig) and its backends: the //! `LoadSpec` contract they trust, its parts' byte runs (`Lattice`), the -//! source request size (`requestSize`), the backend a load runs on -//! (`Backend`) and the memory kinds it takes (`supportsMemory`). +//! source request size (`requestSize`), and which backend and memory kinds +//! a platform takes (`loadsDirect`, `supportsMemory`). const std = @import("std"); const VFS = @import("vfs"); @@ -13,6 +13,7 @@ const Shape = @import("../shape.zig").Shape; const Memory = @import("TensorStore.zig").Memory; const constants = @import("../constants.zig"); const buffered_loader = @import("buffered_loader.zig"); +const direct_loader = @import("direct_loader.zig"); /// The largest source request, and so the largest DMA block: a request is /// a whole number of blocks, at least one. @@ -261,6 +262,22 @@ pub fn checkParts(parts: []const LoadPart, destination_bytes: u64) bool { return covered <= destination_bytes; } +/// Whether `platform` loads through the direct backend. Transfer-manager +/// support and host pinning are independent. CPU implements byte-range +/// transfers from ordinary pages but not DmaMap, so it shares coalescing +/// and bounded blocks without DMA support. The whole-tensor buffered path +/// has neither: on one B70, interleaved HF Qwen3.5-4B loads took 10.9 to +/// 16.3 s direct against about 48 s buffered. Furiosa stages in its +/// plugin's `.host_pinned` buffers (SDK dma-buf memory, which the SDK +/// transfers without pinning pages per request). TPU, neuron and metal +/// stay buffered until their transfer path is checked on hardware. +pub fn loadsDirect(platform: *const Platform) bool { + return switch (platform.target) { + .cuda, .rocm, .oneapi, .cpu, .furiosa => true, + .tpu, .neuron, .metal => false, + }; +} + /// Whether the loader can deliver into `kind` on `platform`: device memory, /// on every platform. The host kinds need a backend that writes host /// memory in place. @@ -269,9 +286,9 @@ pub fn supportsMemory(platform: *const Platform, kind: Memory) bool { return kind == .default; } -/// The transfer implementation a load runs on: the buffered backend, which -/// places each whole tensor with `Buffer.from`. +/// The transfer implementation selected for the platform (`loadsDirect`). pub const Backend = union(enum) { + direct: *direct_loader.Loader, buffered: *buffered_loader.Loader, pub fn init( @@ -280,7 +297,10 @@ pub const Backend = union(enum) { platform: *const Platform, opts: Options, ) !Backend { - return .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, opts) }; + return if (loadsDirect(platform)) + .{ .direct = try direct_loader.Loader.create(allocator, io, platform, opts) } + else + .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, opts) }; } /// Publishes `specs`, and completes one item of `progress` per spec. @@ -318,6 +338,7 @@ pub const Backend = union(enum) { /// A published batch of the backend that published it. pub const Submission = union(enum) { + direct: *direct_loader.Batch, buffered: *buffered_loader.Batch, }; diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 410d2490..3c268e86 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -1,6 +1,7 @@ //! Direct loading follows a batch from per-file planning through FIFO claims, //! source reads and DMA completion. Runtime ownership and synchronization -//! live here. +//! live here. CUDA, ROCm, oneAPI, Furiosa and CPU load this way +//! (`backend.loadsDirect`). //! //! Sizing, fixed by `Loader.create`. The DMA block is the options' one //! (`backend.Options.dmaBlockSizes`); each device's DMA budget is diff --git a/zml/io/host_memory.zig b/zml/io/host_memory.zig index 18061970..2a9553b0 100644 --- a/zml/io/host_memory.zig +++ b/zml/io/host_memory.zig @@ -5,8 +5,9 @@ //! //! The memory depends on the target: CUDA and oneAPI take page-backed arenas //! registered with PJRT (DmaMap), ROCm and Furiosa PJRT `pinned_host` -//! buffers, and CPU plain pages, unregistered. ROCm balances whole arenas -//! across its devices' host nodes. +//! buffers, and CPU plain pages, unregistered. The arenas' kind and the +//! transfer path are separate decisions: CPU loads through the direct +//! backend too. ROCm balances whole arenas across its devices' host nodes. //! `Workspace.mapped_bytes_ceiling` (16 GiB) is a safety guard, not a //! target. A failed allocation or registration, or a growth past the //! ceiling, is logged with its cause and returns `OutOfMemory`. diff --git a/zml/io/loader.zig b/zml/io/loader.zig index 8bcfd1d7..a5755a8e 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -43,6 +43,7 @@ const Tensor = @import("../tensor.zig").Tensor; const testing = @import("../testing.zig"); const backend = @import("backend.zig"); const Backend = backend.Backend; +const buffered_loader = @import("buffered_loader.zig"); const LoadSpec = backend.LoadSpec; const load_log = @import("log.zig").load; const TensorStore = @import("TensorStore.zig"); @@ -53,11 +54,13 @@ const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; /// once. /// /// What it borrows. The platform outlives the loader and every buffer it -/// loads. `io` outlives the loader: the backend's tasks run on it and open -/// the source files through it. `start` borrows the store, the buffers and -/// the progress node until `wait` or `cancel` returns, and reads the model -/// during the call only. The loaded buffers are the caller's and outlive -/// the loader. +/// loads. `io`, and the VFS behind it, outlive the loader: its tasks run on +/// `io`, and the direct backend opens each source file through it and +/// keeps it open until `deinit`, under a copy of the file's path, so +/// nothing of the store is kept past `wait`. `start` borrows the store, the +/// buffers and the progress node until `wait` or `cancel` returns, and +/// reads the model during the call only. The loaded buffers are the +/// caller's and outlive the loader. /// /// Start and wait serially on the owning task; source reads and transfers /// run concurrently inside the backend, compiles on tasks of their own, or @@ -78,10 +81,12 @@ const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; /// can act on: a missing checkpoint file is `FileNotFound`; a denied one /// `AccessDenied` locally and `PermissionDenied` remotely; a file that ends /// inside a tensor `UnexpectedEndOfFile`, because it changed after its -/// header was read; host memory the backend cannot allocate `OutOfMemory`; -/// device memory that runs out PJRT's `ResourceExhausted`; `cancel` -/// `Canceled`. The rest (PJRT, the compiler, a remote that failed) passes -/// through as its source names it. +/// header was read; host memory the backend cannot allocate, pin or fit +/// under its mapped ceiling `OutOfMemory`; device memory that runs out +/// PJRT's `ResourceExhausted`; an io that cannot run the loader's tasks +/// concurrently `ConcurrencyUnavailable`; `cancel` `Canceled`. The rest +/// (PJRT, the compiler, a remote that failed) passes through as its source +/// names it. pub const Loader = struct { allocator: std.mem.Allocator, io: std.Io, @@ -173,9 +178,9 @@ pub const Loader = struct { } /// Cancels `init`'s task when no `start` awaited it, destroying what it - /// built, and destroys the backend: its workers and its host staging. - /// The buffers it loaded stay the caller's. Panics on loads neither - /// waited for nor canceled. + /// built, and destroys the backend: its host memory, its workers and the + /// files it keeps open, closed through `io`. The buffers it loaded stay + /// the caller's. Panics on loads neither waited for nor canceled. pub fn deinit(self: *Loader) void { if (self.loadsStarted()) std.debug.panic("Loader.deinit with loads started, whose store and buffers may be freed already: add `errdefer loader.cancel();` after `start`", .{}); self.queued.deinit(self.allocator); @@ -206,12 +211,13 @@ pub const Loader = struct { /// Every tensor and program input is checked once, before anything is /// published: a tensor `store` did not bind, a source that reads no /// bytes (an empty checkpoint tensor among them), a source that this - /// loader cannot assemble (a run that starts or ends inside a byte, or + /// loader cannot assemble (a run that starts or ends inside a byte; /// strided runs: a transpose, a broadcast, an interleave or interior - /// padding), a sub-byte last dimension that does not pack into whole - /// bytes, a placement its mesh does not divide or on devices the - /// platform lacks, and a memory kind the platform cannot take are - /// `error.InvalidBinding`. + /// padding; or, on a platform that loads through the direct backend + /// (`backend.loadsDirect`), bytes no piece covers: padding), a sub-byte + /// last dimension that does not pack into whole bytes, a placement its + /// mesh does not divide or on devices the platform lacks, and a memory + /// kind the platform cannot take are `error.InvalidBinding`. /// /// `buffers` is `mem.bufferize` of `model`, whose buffers are empty. A /// tree of another structure (other union arms, optionals or slice @@ -757,6 +763,15 @@ fn prepareSpec( load_log.err("{f} from {s}: its parts leave their sources or its {d} bytes, pass the largest file offset, or come out of order", .{ target, key, shape.byteSize() }); return error.InvalidBinding; } + // The direct backend writes the bytes its parts cover, and no others. + if (backend.loadsDirect(platform)) { + var covered: u64 = 0; + for (parts) |part| covered += part.lattice.width; + if (covered != shape.byteSize()) { + load_log.err("{f} from {s}: {d} of its {d} bytes covered by no part (padding), which this loader does not zero on {t}", .{ target, key, shape.byteSize() - covered, shape.byteSize(), platform.target }); + return error.InvalidBinding; + } + } return .{ .shape = shape, .memory = memory, .output = output, .parts = parts }; } @@ -1215,9 +1230,9 @@ test "a failed set-up fails the first start, and deinit frees the rest" { var fixture: LoaderTestFixture = undefined; try fixture.init(std.testing.allocator, io); defer fixture.deinit(); - // `init` allocates nothing; the set-up's first allocation, inside the - // backend, fails. - var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 0 }); + // `init` allocates nothing; the set-up's third allocation, inside the + // backend, fails, so the backend unwinds what it built before it. + var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 2 }); var loader = try Loader.init(failing.allocator(), io, fixture.platform, .{ .read_parallelism = 2 }); defer loader.deinit(); var output: Buffer = undefined; @@ -1298,7 +1313,8 @@ test "a rejected binding or a program of the wrong output fails the loader" { // No binding, an empty source or program input, a mesh of other // devices, a host kind the platform cannot take, a broadcast's strided - // runs, and a program of the wrong shape or placement. + // runs, padding where the direct backend loads, and a program of the + // wrong shape or placement. const Case = struct { tensor: Tensor, at_wait: bool = false, skip: bool = false }; const cases = [_]Case{ .{ .tensor = Tensor.fromShape(fixture.value.shape()) }, @@ -1309,6 +1325,7 @@ test "a rejected binding or a program of the wrong output fails the loader" { .{ .tensor = fixture.computed("value", foreign[2], LoaderTestFixture.Identity{}) }, .{ .tensor = pinned, .skip = backend.supportsMemory(platform, .host_pinned) }, .{ .tensor = fixture.store.view().source("value", .{.x}).broad(Shape.init(.{ .y = 2, .x = 4 }, .u8)).withPartitioning(.replicated, .{}).bind() }, + .{ .tensor = fixture.store.view().source("second", .{.x}).pad(0, .{ .x = TensorStore.Source.Pad{ .high = 2 } }).withPartitioning(.replicated, .{}).bind(), .skip = !backend.loadsDirect(platform) }, .{ .tensor = fixture.computed("value", fixture.value, Rows{}), .at_wait = true }, .{ .tensor = fixture.computed("value", misplaced, LoaderTestFixture.Identity{}), .at_wait = true, .skip = !splits }, }; @@ -1418,3 +1435,60 @@ test "a program takes its inputs from sources, in order" { // { 1, 2, 7, 8 } - { 5, 6, 7, 8 }, wrapping. try testing.expectShards(allocator, io, &output, &.{ 252, 252, 0, 0 }, .default); } + +/// A loader on the buffered backend, whichever backend `platform` loads +/// through (`backend.loadsDirect`), its set-up done in place: CPU loads +/// through the direct backend, and this keeps the front end over the +/// buffered one under test. +fn initBuffered(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Loader.Options) !Loader { + const backend_opts = try backendOptions(opts); + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .setup = null, + .backend = .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, backend_opts) }, + .program_group_bytes = opts.program_group_bytes, + }; +} + +test "the front end loads through the buffered backend and fails with it" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try initBuffered(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + // A tensor, one assembled from value's first half and second's last + // half, and a program tensor, in a group of its own. + const Source = TensorStore.Source; + const view = fixture.store.view(); + const Model = struct { value: Tensor, assembled: Tensor, computed: Tensor }; + const model: Model = .{ + .value = fixture.value, + .assembled = Source.concatenate(&.{ + view.source("value", .{.x}).slice(.x, .{ .end = 2 }), + view.source("second", .{.x}).slice(.x, .{ .start = 2 }), + }, .x).withPartitioning(.replicated, .{}).bind(), + .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Add{ .value = 1 }), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &buffers.assembled, &.{ 1, 2, 7, 8 }, .default); + try testing.expectShards(allocator, io, &buffers.computed, &.{ 2, 3, 4, 5 }, .default); + + // A missing file fails the loader, and every later call with it. + var broken: Buffer = undefined; + try loader.start(&fixture.missing, &broken, &fixture.store, .none); + defer broken.deinit(); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + try std.testing.expectEqual(0, broken._shards.len); + try std.testing.expectError(error.FileNotFound, loader.start(&fixture.value, &broken, &fixture.store, .none)); +} From b289b9cd2b2dd5505adfc70e4d618a8b3cf30510 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:44:22 +0100 Subject: [PATCH 076/103] zml/io: screen DMA blocks on device 0 Adds zml/io/dma_calibration.zig, which picks the direct backend's DMA block once per loader. calibrate maps one ring for the largest candidate in the loader's workspace, where it stays as the load's first pinned capacity, and opens one asynchronous host-to-device transfer manager on device 0, sliced per candidate. Each candidate (2 to 32 MiB by default) is warmed, then measured over three rounds of fixed windows (at least 2 ms and 32 transfers, depth transfers in flight, rotated order), and pick returns the smallest block whose median rate is within 8% of the best. The windows' workers wait behind a start gate and are joined on every exit, a canceled sleep included. CPU returns the smallest candidate without measuring: its transfers are memcpys, and a load takes the same time at every block size. The direct backend does not call it yet. Co-Authored-By: Claude Opus 5.5 --- zml/BUILD.bazel | 1 + zml/io.zig | 1 + zml/io/dma_calibration.zig | 465 +++++++++++++++++++++++++++++++++++++ 3 files changed, 467 insertions(+) create mode 100644 zml/io/dma_calibration.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 4e29611d..bac383e7 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -92,6 +92,7 @@ zig_library( "io/backend.zig", "io/buffered_loader.zig", "io/direct_loader.zig", + "io/dma_calibration.zig", "io/host_memory.zig", "io/loader.zig", "io/log.zig", diff --git a/zml/io.zig b/zml/io.zig index 3fce4fe9..f8d10103 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -33,6 +33,7 @@ test { _ = @import("io/backend.zig"); _ = @import("io/buffered_loader.zig"); _ = @import("io/direct_loader.zig"); + _ = @import("io/dma_calibration.zig"); _ = @import("io/host_memory.zig"); _ = @import("io/loader.zig"); _ = @import("io/log.zig"); diff --git a/zml/io/dma_calibration.zig b/zml/io/dma_calibration.zig new file mode 100644 index 00000000..5eba88e8 --- /dev/null +++ b/zml/io/dma_calibration.zig @@ -0,0 +1,465 @@ +//! DMA sizing for the direct backend: the block size, decided once when the +//! backend is created. It screens `default_block_sizes` (2 to 32 MiB), or +//! the one block the backend's options pin (`backend.Options.dmaBlockSizes`), +//! on one representative device, at `depth` transfers per device and over +//! fixed windows, and keeps the smallest block within 8% of the measured +//! peak. The ring it measures in stays mapped as the load's first pinned +//! capacity. CPU returns the smallest candidate without measuring: its +//! plugin's transfer is a memcpy, and a load takes the same time at every +//! block size. The buffered backend has nothing to size. +const std = @import("std"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const Platform = @import("../platform.zig").Platform; +const host_memory = @import("host_memory.zig"); +const log = @import("log.zig").io; + +/// Blocks in flight per device: the screen runs `depth` transfers (the +/// default `Options.block_parallelism`), and the direct backend's DMA +/// budget is `depth` blocks per device, in bytes (`DevicePump.hasRoom` in +/// direct_loader.zig). Eight repeatedly won or tied on MI300X; wider stages +/// mostly increased callback latency and pinned memory, and an adaptive DMA +/// width added state with little load benefit. At 32, four GB300 loaded +/// DeepSeek 9% faster for 2.5x the pinned memory. +pub const depth: usize = 8; + +/// Screened, not fixed: eight MI300X loaded replicated Gemma in 7.83 s at +/// 16 MiB against 10.69 s at 8, whereas one B70 lost 10.5% at 16 MiB. +pub const default_block_sizes = [_]usize{ + 2 * 1024 * 1024, + 4 * 1024 * 1024, + 8 * 1024 * 1024, + 16 * 1024 * 1024, + 32 * 1024 * 1024, +}; + +/// The screen's knobs. The loader sets only `block_sizes` and keeps the +/// other defaults. +pub const Options = struct { + block_sizes: []const usize = &default_block_sizes, + /// Transfers in flight per device in the screen. The loader keeps + /// `depth`. + block_parallelism: usize = depth, + /// A screen window runs for at least this long and, unless the target is + /// zero, until the representative device completes the transfer target. + /// On eight MI300X, 2 ms/32-transfer windows calibrate in 0.96 s against + /// 4.83 s at 10 ms/128, selecting the same block. + minimum_duration: std.Io.Duration = .fromMilliseconds(2), + minimum_transfers: u64 = 32, + /// Prefer the smallest block within this tolerance of the measured peak + /// transfer throughput. + block_selection_tolerance: f64 = 0.08, +}; + +/// Measures one representative device into the direct loader's workspace +/// and returns the block it selects; CPU returns the smallest without +/// measuring. On a platform mixing device kinds, device 0's block serves +/// every device: it is a tuning choice, valid for all of them. Each block +/// of `opts` is nonzero and at most `backend.max_read_request_size`, so the +/// ring fits the workspace's mapped ceiling. The rate the `dma_bench` line +/// logs is that one device's synthetic H2D rate, not the platform's nor a +/// prediction of a checkpoint's load speed. +pub fn calibrate( + workspace: *host_memory.Workspace, + platform: *const Platform, + opts: Options, +) !usize { + // Every candidate loads alike on CPU: four PJRT CPU devices loaded warm + // sharded Llama in 1.55-1.60 s at 2, 8 and 16 MiB, so the smallest, + // which maps the least, is taken unmeasured and the loader grows its + // own arenas. + if (platform.target == .cpu) return std.mem.min(usize, opts.block_sizes); + + const allocator = workspace.allocator; + const io = workspace.io; + const start: std.Io.Timestamp = .now(io, .awake); + + var max_total_bytes: usize = 0; + for (opts.block_sizes) |block_size| { + max_total_bytes = @max(max_total_bytes, block_size * opts.block_parallelism); + } + // Map one ring for the largest candidate at the fixed transfer width; + // every candidate reuses it, and the load keeps it. + const calibration_source = try workspace.allocate(max_total_bytes); + + var session = try Session.init(allocator, io, platform, opts.block_sizes, opts.block_parallelism); + defer session.deinit(); + + const candidates = try allocator.alloc(Candidate, opts.block_sizes.len); + defer allocator.free(candidates); + for (candidates, opts.block_sizes, 0..) |*candidate, block_size, index| { + candidate.* = .{ .block_size = block_size, .buffer_offset = index * opts.block_parallelism }; + } + try measureAllCandidates(&session, opts, candidates, calibration_source); + const winner = candidates[try selectCandidate(&session, opts, candidates)]; + + log.debug("dma_bench platform={s} devices={d} kind=\"{s}\" block_bytes={d} parallelism={d} measured_gib_s={d:.3} duration={f} retained_mapped_bytes={d}", .{ + @tagName(platform.target), + platform.devices.len, + platform.devices[0].kind(), + winner.block_size, + opts.block_parallelism, + winner.median().bytesPerSecond() / (1024 * 1024 * 1024), + start.durationTo(.now(io, .awake)), + workspace.mappedBytes(), + }); + return winner.block_size; +} + +const sample_count = 3; + +fn measureAllCandidates( + session: *Session, + opts: Options, + candidates: []Candidate, + source: []const u8, +) !void { + for (0..sample_count) |round| { + for (0..candidates.len) |i| { + const candidate = &candidates[(i + round) % candidates.len]; + candidate.metrics[round] = try runWindow( + session.io, + session, + candidate, + source, + opts.minimum_duration, + opts.minimum_transfers, + ); + } + } +} + +/// Returns the index of the selected candidate (`pick`), over each +/// candidate's median ratio to the peak. +fn selectCandidate( + session: *Session, + opts: Options, + candidates: []Candidate, +) !usize { + const ratios = try session.allocator.alloc(f64, candidates.len); + defer session.allocator.free(ratios); + + var peak_rate: f64 = 0; + for (candidates) |candidate| peak_rate = @max(peak_rate, candidate.median().bytesPerSecond()); + for (candidates, ratios) |candidate, *ratio| { + ratio.* = if (peak_rate == 0) 0 else candidate.median().bytesPerSecond() / peak_rate; + } + return pick(candidates, ratios, opts.block_selection_tolerance); +} + +/// The smallest block whose ratio is within `tolerance` of the best ratio. +fn pick(candidates: []const Candidate, ratios: []const f64, tolerance: f64) usize { + var best_index: usize = 0; + for (ratios, 0..) |ratio, index| { + if (ratio > ratios[best_index]) best_index = index; + } + const floor = ratios[best_index] * (1.0 - tolerance); + var selected_index = best_index; + for (candidates, ratios, 0..) |candidate, ratio, index| { + if (ratio >= floor and candidate.block_size < candidates[selected_index].block_size) + selected_index = index; + } + return selected_index; +} + +/// Runs one window of `candidate`'s transfers, `session.parallelism` at a +/// time. `io` sleeps and joins the window; the workers transfer on the +/// session's. +fn runWindow( + io: std.Io, + session: *Session, + candidate: *Candidate, + source: []const u8, + minimum_duration: std.Io.Duration, + minimum_transfers: u64, +) !Measurement { + const Worker = struct { + session: *Session, + buffer_index: usize, + block: []const u8, + metrics: *Counters, + ready: *std.atomic.Value(usize), + start: *std.Io.Event, + stop: *std.atomic.Value(bool), + + fn run(self: @This()) void { + _ = self.ready.fetchAdd(1, .release); + self.start.waitUncancelable(self.session.io); + while (!self.stop.load(.acquire)) { + self.session.transfer(self.buffer_index, self.block, self.metrics); + if (self.session.first_error.get() != null) return; + } + } + }; + + var metrics: Counters = .{}; + var ready: std.atomic.Value(usize) = .init(0); + var start: std.Io.Event = .unset; + var stop: std.atomic.Value(bool) = .init(false); + var group: std.Io.Group = .init; + defer { + // Workers borrow this window's stack. Every exit, including a canceled + // sleep or partial spawn, must release the start gate and join them. + stop.store(true, .release); + start.set(io); + group.await(io) catch {}; + } + for (0..session.parallelism) |slot| { + try group.concurrent(io, Worker.run, .{Worker{ + .session = session, + .buffer_index = candidate.buffer_offset + slot, + .block = candidate.block(source, slot), + .metrics = &metrics, + .ready = &ready, + .start = &start, + .stop = &stop, + }}); + } + + try session.warmup(candidate, source); + + while (ready.load(.acquire) != session.parallelism) try io.sleep(.fromMicroseconds(100), .awake); + const start_at: std.Io.Timestamp = .now(io, .awake); + start.set(io); + while (true) { + const elapsed = start_at.durationTo(.now(io, .awake)); + if (session.first_error.get()) |err| return err; + // is window complete + if (elapsed.nanoseconds >= minimum_duration.nanoseconds and metrics.transfers.load(.acquire) >= minimum_transfers) break; + try io.sleep(.fromMilliseconds(1), .awake); + } + stop.store(true, .release); + try group.await(io); + const elapsed = start_at.durationTo(.now(io, .awake)); + if (session.first_error.get()) |err| return err; + return .{ .bytes = metrics.bytes.load(.acquire), .elapsed = elapsed }; +} + +/// One block size and its screen measurements. Its slots are the session's +/// buffers from `buffer_offset` on, one per transfer in flight. +const Candidate = struct { + block_size: usize, + buffer_offset: usize, + /// Slots warmed so far (`Session.warmup`). + warmed_buffers: usize = 0, + + /// One window per screen round (`measureAllCandidates`). + metrics: [sample_count]Measurement = undefined, + + fn median(self: Candidate) Measurement { + var sorted = self.metrics; + std.mem.sort(Measurement, &sorted, {}, struct { + fn lessThan(_: void, lhs: Measurement, rhs: Measurement) bool { + return lhs.bytesPerSecond() < rhs.bytesPerSecond(); + } + }.lessThan); + return sorted[sorted.len / 2]; + } + + /// The ring's bytes that `slot` transfers. + fn block(self: Candidate, source: []const u8, slot: usize) []const u8 { + return source[slot * self.block_size ..][0..self.block_size]; + } +}; + +/// One transfer manager for the whole screen, sliced per candidate, and the +/// screen's first transfer error, which ends it. +const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + manager: *pjrt.AsyncHostToDeviceTransferManager, + buffers: []*pjrt.Buffer, + /// Each candidate's slots, and the transfers a window runs at a time. + parallelism: usize, + first_error: stdx.FirstError(pjrt.ApiError) = .{}, + + fn transfer(self: *Session, buffer_index: usize, block: []const u8, metrics: ?*Counters) void { + const api = self.platform.pjrt_api; + const event = self.manager.transferData(api, buffer_index, block, 0, false) catch |err| { + _ = self.first_error.set(err); + return; + }; + defer event.deinit(api); + event.await(api, self.io) catch |err| { + _ = self.first_error.set(err); + return; + }; + if (metrics) |output| { + _ = output.bytes.fetchAdd(@intCast(block.len), .monotonic); + _ = output.transfers.fetchAdd(1, .monotonic); + } + } + + /// Transfers twice into each of `candidate`'s slots not yet warmed. + fn warmup(self: *Session, candidate: *Candidate, source: []const u8) !void { + while (candidate.warmed_buffers < self.parallelism) : (candidate.warmed_buffers += 1) { + const slot = candidate.warmed_buffers; + self.transfer(candidate.buffer_offset + slot, candidate.block(source, slot), null); + self.transfer(candidate.buffer_offset + slot, candidate.block(source, slot), null); + if (self.first_error.get()) |err| return err; + } + } + + fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + block_sizes: []const usize, + parallelism: usize, + ) !Session { + const buffer_count = std.math.mul(usize, block_sizes.len, parallelism) catch return error.OutOfMemory; + const dims = try allocator.alloc([1]i64, block_sizes.len); + defer allocator.free(dims); + const specs = try allocator.alloc(pjrt.ShapeSpec, buffer_count); + defer allocator.free(specs); + for (block_sizes, dims, 0..) |block_size, *dim, index| { + dim.* = .{@intCast(block_size)}; + @memset(specs[index * parallelism ..][0..parallelism], pjrt.ShapeSpec.init(dim, .u8)); + } + const buffers = try allocator.alloc(*pjrt.Buffer, buffer_count); + errdefer allocator.free(buffers); + const manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ + .shape_specs = specs, + .memory = platform.devices[0].memory(.default).?.pjrt_memory, + }); + var retrieved: usize = 0; + errdefer { + manager.deinit(platform.pjrt_api); + for (buffers[0..retrieved]) |buffer| buffer.deinit(platform.pjrt_api); + } + for (buffers, 0..) |*buffer, index| { + buffer.* = try manager.retrieveBuffer(platform.pjrt_api, index); + retrieved += 1; + } + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .manager = manager, + .buffers = buffers, + .parallelism = parallelism, + }; + } + + fn deinit(self: *Session) void { + // XLA commit 1b19ae012aa67426658f7ca3c1503bb781c863a9 switched GPU + // transfers to CommonAsyncHostToDeviceTransferManager, whose destructor + // waits for outstanding transfers and marks unfinished buffers as errored. + // We discard these buffers, so no final is_last_transfer=true copy is needed. + self.manager.deinit(self.platform.pjrt_api); + for (self.buffers) |buffer| buffer.deinit(self.platform.pjrt_api); + self.allocator.free(self.buffers); + self.* = undefined; + } +}; + +const Counters = struct { + bytes: std.atomic.Value(u64) = .init(0), + transfers: std.atomic.Value(u64) = .init(0), +}; + +const Measurement = struct { + bytes: u64, + elapsed: std.Io.Duration, + + fn bytesPerSecond(self: Measurement) f64 { + if (self.elapsed.nanoseconds <= 0) return 0; + return @as(f64, @floatFromInt(self.bytes)) * std.time.ns_per_s / + @as(f64, @floatFromInt(self.elapsed.nanoseconds)); + } +}; + +test "DMA benchmark selection uses medians and prefers the smallest near-peak value" { + // The selector only allocates its ratios: the session supplies only its + // allocator. + var session: Session = .{ + .allocator = std.testing.allocator, + .io = undefined, + .platform = undefined, + .manager = undefined, + .buffers = undefined, + .parallelism = undefined, + }; + const opts: Options = .{ .block_selection_tolerance = 0.05 }; + + var candidates = [_]Candidate{ + .{ .block_size = 2, .buffer_offset = undefined }, + .{ .block_size = 4, .buffer_offset = undefined }, + .{ .block_size = 8, .buffer_offset = undefined }, + }; + const rates = [_][sample_count]u64{ + .{ 60, 10, 62 }, + .{ 98, 99, 97 }, + .{ 100, 101, 99 }, + }; + for (&candidates, rates) |*candidate, candidate_rates| { + for (&candidate.metrics, candidate_rates) |*metric, rate| { + metric.* = .{ .bytes = rate, .elapsed = .fromSeconds(1) }; + } + } + const winner = candidates[try selectCandidate(&session, opts, &candidates)]; + try std.testing.expectEqual(@as(usize, 4), winner.block_size); + try std.testing.expectEqual(@as(f64, 98), winner.median().bytesPerSecond()); +} + +test "DMA benchmark picks the smallest block within the tolerance of the best ratio" { + const candidates = [_]Candidate{ + .{ .block_size = 16 * 1024 * 1024, .buffer_offset = undefined }, + .{ .block_size = 32 * 1024 * 1024, .buffer_offset = undefined }, + }; + // 32 MiB is the peak; 16 MiB within the tolerance below it, then past it. + try std.testing.expectEqual(0, pick(&candidates, &.{ 0.93, 1.0 }, 0.08)); + try std.testing.expectEqual(1, pick(&candidates, &.{ 0.91, 1.0 }, 0.08)); +} + +test "DMA benchmark shares one manager across candidate sizes and slots" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.auto(allocator, io, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + var session = try Session.init(allocator, io, platform, &.{ 8, 16 }, 2); + defer session.deinit(); + const source: [32]u8 = @splat(0); + for ([_]usize{ 8, 16 }, 0..) |block_size, index| { + var candidate: Candidate = .{ .block_size = block_size, .buffer_offset = index * 2 }; + try session.warmup(&candidate, &source); + try std.testing.expectEqual(2, candidate.warmed_buffers); + } + try std.testing.expectEqual(null, session.first_error.get()); +} + +test "DMA benchmark cancellation drains transfer workers" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.auto(allocator, io, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + var source: [16]u8 = @splat(0); + var session = try Session.init(allocator, io, platform, &.{source.len}, 1); + defer session.deinit(); + var candidate: Candidate = .{ .block_size = source.len, .buffer_offset = 0 }; + var vtable = io.vtable.*; + vtable.sleep = struct { + fn sleep(_: ?*anyopaque, _: std.Io.Timeout) std.Io.Cancelable!void { + return error.Canceled; + } + }.sleep; + const canceled_io: std.Io = .{ .userdata = io.userdata, .vtable = &vtable }; + + // A cancellation can arrive while workers wait for the start gate or + // while they transfer. In either case they must finish before teardown. + try std.testing.expectError(error.Canceled, runWindow( + canceled_io, + &session, + &candidate, + &source, + .max, + 0, + )); +} From 6a04d00de3cd2336072b2892d66d0a3695cc29ee Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:46:46 +0100 Subject: [PATCH 077/103] zml/io: confirm borderline candidates A short screen window is noisy, and a candidate near the selection floor could be picked or dropped by chance. A candidate is now borderline when its rounds disagree on whether it is near the peak, or when its median ratio lies within 0.02 of the tolerance's floor. Each borderline candidate runs three paired 25 ms, 256-transfer windows against the peak, alternating which goes first, and its median ratio from those windows replaces its screen ratio before pick. A confirmation can rate a larger block above the screen's peak, so pick takes the best ratio from either source. Co-Authored-By: Claude Opus 5.5 --- zml/io/dma_calibration.zig | 138 ++++++++++++++++++++++++++++++++++--- 1 file changed, 128 insertions(+), 10 deletions(-) diff --git a/zml/io/dma_calibration.zig b/zml/io/dma_calibration.zig index 5eba88e8..962224e1 100644 --- a/zml/io/dma_calibration.zig +++ b/zml/io/dma_calibration.zig @@ -3,7 +3,8 @@ //! the one block the backend's options pin (`backend.Options.dmaBlockSizes`), //! on one representative device, at `depth` transfers per device and over //! fixed windows, and keeps the smallest block within 8% of the measured -//! peak. The ring it measures in stays mapped as the load's first pinned +//! peak; borderline candidates get longer, alternating confirmation +//! windows. The ring it measures in stays mapped as the load's first pinned //! capacity. CPU returns the smallest candidate without measuring: its //! plugin's transfer is a memcpy, and a load takes the same time at every //! block size. The buffered backend has nothing to size. @@ -45,7 +46,9 @@ pub const Options = struct { /// A screen window runs for at least this long and, unless the target is /// zero, until the representative device completes the transfer target. /// On eight MI300X, 2 ms/32-transfer windows calibrate in 0.96 s against - /// 4.83 s at 10 ms/128, selecting the same block. + /// 4.83 s at 10 ms/128, selecting the same block. Short screens sometimes + /// selected the wrong block under noise, hence the longer borderline + /// confirmation windows (`confirmation_duration`). minimum_duration: std.Io.Duration = .fromMilliseconds(2), minimum_transfers: u64 = 32, /// Prefer the smallest block within this tolerance of the measured peak @@ -53,6 +56,14 @@ pub const Options = struct { block_selection_tolerance: f64 = 0.08, }; +/// Borderline block candidates receive longer paired windows, alternating +/// which runs first. A candidate is borderline when its rounds disagree on +/// whether it is near the peak, or when its median ratio to the peak lies +/// within `confirmation_margin` of the tolerance's floor. +const confirmation_duration: std.Io.Duration = .fromMilliseconds(25); +const confirmation_minimum_transfers: u64 = 256; +const confirmation_margin: f64 = 0.02; + /// Measures one representative device into the direct loader's workspace /// and returns the block it selects; CPU returns the smallest without /// measuring. On a platform mixing device kinds, device 0's block serves @@ -93,7 +104,7 @@ pub fn calibrate( candidate.* = .{ .block_size = block_size, .buffer_offset = index * opts.block_parallelism }; } try measureAllCandidates(&session, opts, candidates, calibration_source); - const winner = candidates[try selectCandidate(&session, opts, candidates)]; + const winner = candidates[try selectCandidate(&session, opts, candidates, calibration_source)]; log.debug("dma_bench platform={s} devices={d} kind=\"{s}\" block_bytes={d} parallelism={d} measured_gib_s={d:.3} duration={f} retained_mapped_bytes={d}", .{ @tagName(platform.target), @@ -132,24 +143,41 @@ fn measureAllCandidates( } /// Returns the index of the selected candidate (`pick`), over each -/// candidate's median ratio to the peak. +/// candidate's median ratio to the peak, or its confirmation's when it is +/// borderline. fn selectCandidate( session: *Session, opts: Options, candidates: []Candidate, + source: []const u8, ) !usize { const ratios = try session.allocator.alloc(f64, candidates.len); defer session.allocator.free(ratios); - var peak_rate: f64 = 0; - for (candidates) |candidate| peak_rate = @max(peak_rate, candidate.median().bytesPerSecond()); + var peak_index: usize = 0; + var peak_rate = candidates[0].median().bytesPerSecond(); + for (candidates[1..], 1..) |candidate, index| { + const rate = candidate.median().bytesPerSecond(); + if (rate > peak_rate) { + peak_index = index; + peak_rate = rate; + } + } for (candidates, ratios) |candidate, *ratio| { ratio.* = if (peak_rate == 0) 0 else candidate.median().bytesPerSecond() / peak_rate; } + + // A borderline candidate's ratio is its confirmation's against the peak. + for (candidates, ratios, 0..) |*candidate, *ratio, index| { + if (index != peak_index and isBorderline(candidates, index, ratio.*, opts.block_selection_tolerance)) + ratio.* = try confirmCandidateAgainstPeak(session, candidate, &candidates[peak_index], source); + } return pick(candidates, ratios, opts.block_selection_tolerance); } -/// The smallest block whose ratio is within `tolerance` of the best ratio. +/// The smallest block whose ratio is within `tolerance` of the best ratio, +/// screened or confirmed. A confirmation can rate a block above the screen's +/// peak, so the best ratio may belong to a larger block. fn pick(candidates: []const Candidate, ratios: []const f64, tolerance: f64) usize { var best_index: usize = 0; for (ratios, 0..) |ratio, index| { @@ -164,6 +192,59 @@ fn pick(candidates: []const Candidate, ratios: []const f64, tolerance: f64) usiz return selected_index; } +/// Whether `candidates[index]`, at screen `ratio` to the peak, gets +/// confirmation windows (`confirmation_margin`). +fn isBorderline(candidates: []const Candidate, index: usize, ratio: f64, tolerance: f64) bool { + // Whether the candidate is near-peak in one round but fails in others. + var qualified_once = false; + var rejected_once = false; + for (candidates[index].metrics, 0..) |metric, round| { + var round_peak_rate: f64 = 0; + for (candidates) |round_candidate| { + round_peak_rate = @max(round_peak_rate, round_candidate.metrics[round].bytesPerSecond()); + } + const round_ratio = if (round_peak_rate == 0) 0 else metric.bytesPerSecond() / round_peak_rate; + if (round_ratio >= 1.0 - tolerance) + qualified_once = true + else + rejected_once = true; + } + return (qualified_once and rejected_once) or @abs(ratio - (1.0 - tolerance)) <= confirmation_margin; +} + +/// The median, over `sample_count` paired windows, of `candidate`'s rate +/// over `peak`'s. +fn confirmCandidateAgainstPeak( + session: *Session, + candidate: *Candidate, + peak: *Candidate, + source: []const u8, +) !f64 { + var ratios: [sample_count]f64 = undefined; + for (&ratios, 0..) |*ratio, repeat| { + const order = if (repeat % 2 == 0) + [_]*Candidate{ candidate, peak } + else + [_]*Candidate{ peak, candidate }; + var candidate_rate: f64 = undefined; + var peak_rate: f64 = undefined; + for (order) |measured| { + const rate = (try runWindow( + session.io, + session, + measured, + source, + confirmation_duration, + confirmation_minimum_transfers, + )).bytesPerSecond(); + if (measured == candidate) candidate_rate = rate else peak_rate = rate; + } + ratio.* = if (peak_rate == 0) 0 else candidate_rate / peak_rate; + } + std.mem.sort(f64, &ratios, {}, std.sort.asc(f64)); + return ratios[ratios.len / 2]; +} + /// Runs one window of `candidate`'s transfers, `session.parallelism` at a /// time. `io` sleeps and joins the window; the workers transfer on the /// session's. @@ -373,8 +454,8 @@ const Measurement = struct { }; test "DMA benchmark selection uses medians and prefers the smallest near-peak value" { - // The selector only allocates its ratios: the session supplies only its - // allocator. + // No candidate below is borderline, so the selector never schedules a + // confirmation window and the session supplies only its allocator. var session: Session = .{ .allocator = std.testing.allocator, .io = undefined, @@ -400,9 +481,23 @@ test "DMA benchmark selection uses medians and prefers the smallest near-peak va metric.* = .{ .bytes = rate, .elapsed = .fromSeconds(1) }; } } - const winner = candidates[try selectCandidate(&session, opts, &candidates)]; + const winner = candidates[try selectCandidate(&session, opts, &candidates, &.{})]; try std.testing.expectEqual(@as(usize, 4), winner.block_size); try std.testing.expectEqual(@as(f64, 98), winner.median().bytesPerSecond()); + + // A larger block near the peak in every round is neither confirmed nor + // picked. + var bimodal = [_]Candidate{ + .{ .block_size = 2, .buffer_offset = undefined }, + .{ .block_size = 4, .buffer_offset = undefined }, + .{ .block_size = 8, .buffer_offset = undefined }, + .{ .block_size = 16, .buffer_offset = undefined }, + }; + const bimodal_rates = [_]u64{ 80, 100, 70, 99 }; + for (&bimodal, bimodal_rates) |*candidate, rate| { + candidate.metrics = @splat(.{ .bytes = rate, .elapsed = .fromSeconds(1) }); + } + try std.testing.expectEqual(@as(usize, 4), bimodal[try selectCandidate(&session, opts, &bimodal, &.{})].block_size); } test "DMA benchmark picks the smallest block within the tolerance of the best ratio" { @@ -413,6 +508,29 @@ test "DMA benchmark picks the smallest block within the tolerance of the best ra // 32 MiB is the peak; 16 MiB within the tolerance below it, then past it. try std.testing.expectEqual(0, pick(&candidates, &.{ 0.93, 1.0 }, 0.08)); try std.testing.expectEqual(1, pick(&candidates, &.{ 0.91, 1.0 }, 0.08)); + // 16 MiB is the screen's peak; a confirmation rates 32 MiB more than the + // tolerance above it, then within it. + try std.testing.expectEqual(1, pick(&candidates, &.{ 1.0, 1.10 }, 0.08)); + try std.testing.expectEqual(0, pick(&candidates, &.{ 1.0, 1.05 }, 0.08)); +} + +test "DMA benchmark confirms a candidate when round qualification disagrees" { + var candidates = [_]Candidate{ + .{ .block_size = 4, .buffer_offset = undefined }, + .{ .block_size = 8, .buffer_offset = undefined }, + }; + const rates = [_][sample_count]u64{ + .{ 99, 80, 70 }, + .{ 100, 100, 100 }, + }; + for (&candidates, rates) |*candidate, candidate_rates| { + for (&candidate.metrics, candidate_rates) |*metric, rate| { + metric.* = .{ .bytes = rate, .elapsed = .fromSeconds(1) }; + } + } + // The median ratio, 0.80, is far from the 0.95 floor: only the rounds + // make it borderline. + try std.testing.expect(isBorderline(&candidates, 0, 0.80, 0.05)); } test "DMA benchmark shares one manager across candidate sizes and slots" { From d6f54bf518d1e1fd4b9995c35e3529f97ebb7e9f Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:47:01 +0100 Subject: [PATCH 078/103] zml/io: size the direct backend from calibration The direct backend now takes its DMA block from dma_calibration.calibrate instead of one fixed 16 MiB block. create calibrates in its own workspace before it sizes anything else, so the ring the screen measured in stays mapped and the pre-growth maps only what it leaves; the debug line reports both. Each device's DMA budget is dma_calibration.depth blocks, which moves there from direct_loader.zig, and backend.Options .dmaBlockSizes returns the default candidates, 2 to 32 MiB, unless the options pin one block. CPU takes the smallest candidate, 2 MiB, which maps a quarter of the pinned set the fixed block did. A failed or canceled calibration fails the set-up, logged once. Co-Authored-By: Claude Opus 5.5 --- zml/io/backend.zig | 18 +++++------- zml/io/direct_loader.zig | 63 ++++++++++++++++++++++------------------ zml/io/host_memory.zig | 19 ++++++------ zml/io/loader.zig | 3 +- 4 files changed, 55 insertions(+), 48 deletions(-) diff --git a/zml/io/backend.zig b/zml/io/backend.zig index 6f6a3502..5f005a21 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -14,6 +14,7 @@ const Memory = @import("TensorStore.zig").Memory; const constants = @import("../constants.zig"); const buffered_loader = @import("buffered_loader.zig"); const direct_loader = @import("direct_loader.zig"); +const dma_calibration = @import("dma_calibration.zig"); /// The largest source request, and so the largest DMA block: a request is /// a whole number of blocks, at least one. @@ -51,18 +52,15 @@ pub const Options = struct { read_parallelism: usize, /// The checkpoint's VFS backend's read hints. read_hints: VFS.ReadHints = .local, - /// The DMA block, else `fixed_dma_block_size`. + /// The DMA block, else one calibration picks. dma_block_size: ?usize = null, /// Whether local files are read directly (`VFS.useDirectIo`). direct_io: VFS.DirectIo = .auto, - /// The DMA block when the options pin none. - pub const fixed_dma_block_size: usize = 16 * 1024 * 1024; - - /// The blocks a backend may use: `dma_block_size`, else - /// `fixed_dma_block_size`. + /// The blocks calibration may select: `dma_block_size`, else + /// `dma_calibration.default_block_sizes`. pub fn dmaBlockSizes(self: *const Options) []const usize { - return if (self.dma_block_size) |*block_size| block_size[0..1] else &.{fixed_dma_block_size}; + return if (self.dma_block_size) |*block_size| block_size[0..1] else &dma_calibration.default_block_sizes; } }; @@ -459,11 +457,11 @@ test "requestSize rounds the read chunk up to whole blocks of whole alignment un try std.testing.expectEqual(3 * 4096, requestSize(4096, 3 * 4096, 4096)); } -test "the options pin a DMA block or take the fixed one, and only a VFS widens reads" { +test "the options pin a DMA block or leave calibration its candidates, and only a VFS widens reads" { const pinned: Options = .{ .read_parallelism = 1, .dma_block_size = 4 << 20 }; try std.testing.expectEqualSlices(usize, &.{4 << 20}, pinned.dmaBlockSizes()); - const fixed: Options = .{ .read_parallelism = 1 }; - try std.testing.expectEqualSlices(usize, &.{Options.fixed_dma_block_size}, fixed.dmaBlockSizes()); + const calibrated: Options = .{ .read_parallelism = 1 }; + try std.testing.expectEqualSlices(usize, &dma_calibration.default_block_sizes, calibrated.dmaBlockSizes()); var filesystem = try VFS.init(std.testing.allocator, std.testing.io); defer filesystem.deinit(); diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 3c268e86..574c1df5 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -3,19 +3,20 @@ //! live here. CUDA, ROCm, oneAPI, Furiosa and CPU load this way //! (`backend.loadsDirect`). //! -//! Sizing, fixed by `Loader.create`. The DMA block is the options' one -//! (`backend.Options.dmaBlockSizes`); each device's DMA budget is -//! `dma_depth` blocks. A source request is the checkpoint's minimum read -//! (`VFS.readHints`) rounded up to whole blocks, one block at least -//! (`backend.requestSize`). The source width is `read_parallelism`, -//! clipped to what the mapped ceiling fits. The workspace then maps, once -//! and before the first read, `width + 1` requests, the DMA reserve and the -//! zero block, and cuts them into blocks (`Workspace.carve`): nothing maps -//! during a load. `width + 1` workers share `width` read permits, so a full -//! width of reads is in flight while one worker queues what it read. The -//! lifecycle credits are the whole pinned set in requests, so transfers -//! that still hold blocks keep the DMA stage fed and never want a block the -//! reads did not leave free. +//! Sizing, fixed by `Loader.create`. Calibration picks the DMA block +//! (dma_calibration.zig) and maps the ring it measures in; each device's +//! DMA budget is `dma_calibration.depth` blocks. A source request is the +//! checkpoint's minimum read (`VFS.readHints`) rounded up to whole blocks, +//! one block at least (`backend.requestSize`). The source width is +//! `read_parallelism`, clipped to what the mapped ceiling fits. The +//! workspace then maps, once and before the first read, what the ring +//! leaves of `width + 1` requests, the DMA reserve and the zero block, and +//! cuts them into blocks (`Workspace.carve`): nothing maps during a load. +//! `width + 1` workers share `width` read permits, so a full width of reads +//! is in flight while one worker queues what it read. The lifecycle credits +//! are the whole pinned set in requests, so transfers that still hold +//! blocks keep the DMA stage fed and never want a block the reads did not +//! leave free. //! //! Planning and claims. `submit` sorts a submission's parts by file and //! offset, plans one file at a time and publishes each plan at once, so the @@ -59,6 +60,7 @@ const BackendOptions = backend.Options; const LoadPart = backend.LoadPart; const LoadSpec = backend.LoadSpec; const ShardGrid = @import("ShardGrid.zig"); +const dma_calibration = @import("dma_calibration.zig"); const Memory = @import("TensorStore.zig").Memory; const host_memory = @import("host_memory.zig"); const load_log = @import("log.zig").load; @@ -74,9 +76,6 @@ const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; /// byte budget. const max_dma_pieces_per_device: usize = 64; -/// Each device's in-flight DMA budget, in blocks (`Dma.DevicePump.hasRoom`). -const dma_depth: usize = 8; - /// The direct DMA backend. Submissions and awaits come from one task at a /// time; the workers and the pumps run concurrently with them. pub const Loader = struct { @@ -110,7 +109,7 @@ pub const Loader = struct { /// (`Dma.enqueue`, `Dma.retireBatch`) they are taken in device /// order; a pump, a completion and `Dma.abortReady` hold one at a time. pumps: []Dma.DevicePump, - /// Per-device in-flight budget: `dma_depth` blocks, in bytes. + /// Per-device in-flight budget: `dma_calibration.depth` blocks, in bytes. dma_budget_bytes: usize, worker_group: std.Io.Group = .init, /// One per source path. Its file opens at its first read and stays @@ -136,15 +135,21 @@ pub const Loader = struct { var workspace: host_memory.Workspace = try .init(allocator, io, platform); errdefer workspace.deinit(); - // The options' block for the whole load; every read is exact. - const block_size = opts.dmaBlockSizes()[0]; + // At debug when canceled (`zml.io.Loader.deinit` cancels a set-up + // no `start` waited for) or out of memory (the workspace logs what + // it fails to map or pin). + const block_size = dma_calibration.calibrate(&workspace, platform, .{ .block_sizes = opts.dmaBlockSizes() }) catch |err| { + load_log.failure(err, "calibrate DMA: target={t}", .{platform.target}); + return err; + }; + // Every read is exact. const request_size = backend.requestSize(opts.read_hints.read_chunk_size, block_size, 0).?; const blocks_per_request = request_size / block_size; // The DMA stage of every device, kept mapped beside the source // working set; at least one maximal source request, so a single // request always fits beside it. const dma_reserve = @max( - dma_depth * platform.devices.len, + dma_calibration.depth * platform.devices.len, Planner.blockCount(backend.max_read_request_size, block_size), ); // One more block for the zeros, which no credit owns. @@ -167,11 +172,13 @@ pub const Loader = struct { // Two arenas, the reserve first: ROCm balances bytes across its host // nodes per allocation, and that split is recorded. const pregrowth_started: std.Io.Timestamp = .now(io, .awake); + const retained_before = workspace.mappedBytes(); // The workspace logs what it fails to map. const blocks = try workspace.carve(block_size, dma_reserve, (width + 1) * blocks_per_request + dma_reserve + zero_blocks); const pregrowth_ns: u64 = @intCast(@max(pregrowth_started.untilNow(io, .awake).nanoseconds, 0)); - load_log.debug("host workspace pregrown: mapped={Bi:.2}, pregrowth_ms={d:.3}", .{ + load_log.debug("host workspace pregrown: retained={Bi:.2}, pregrown={Bi:.2}, pregrowth_ms={d:.3}", .{ workspace.mappedBytes(), + workspace.mappedBytes() - retained_before, @as(f64, @floatFromInt(pregrowth_ns)) / std.time.ns_per_ms, }); std.debug.assert(blocks.len != 0); @@ -195,7 +202,7 @@ pub const Loader = struct { .read_width = width, .credits = credits, .pumps = pumps, - .dma_budget_bytes = dma_depth * block_size, + .dma_budget_bytes = dma_calibration.depth * block_size, .plan_config = .{ .block_size = block_size, .request_size = request_size, @@ -1305,12 +1312,12 @@ const Dma = struct { retired: ?*EventContext = null, /// Whether the next queued piece may go. The budget is in bytes, - /// not submissions: `dma_depth` blocks, and a piece is one tensor's - /// slice of a block, so a model of small tensors needs many more - /// pieces in flight to keep the same bytes moving: eight average - /// DeepSeek pieces occupy only ~18 MiB against the 128 MiB budget of - /// 16 MiB blocks. The piece that crosses the budget is admitted: a - /// device with room always has a piece in flight. + /// not submissions: `dma_calibration.depth` blocks, and a piece is + /// one tensor's slice of a block, so a model of small tensors needs + /// many more pieces in flight to keep the same bytes moving: eight + /// average DeepSeek pieces occupy only ~18 MiB against the 128 MiB + /// budget of 16 MiB blocks. The piece that crosses the budget is + /// admitted: a device with room always has a piece in flight. fn hasRoom(self: *const DevicePump, budget_bytes: usize) bool { return self.active_bytes < budget_bytes and self.active_pieces < max_dma_pieces_per_device; } diff --git a/zml/io/host_memory.zig b/zml/io/host_memory.zig index 2a9553b0..1f684658 100644 --- a/zml/io/host_memory.zig +++ b/zml/io/host_memory.zig @@ -1,7 +1,8 @@ //! Loader-owned host arenas, cut into blocks once. The direct backend owns -//! its workspace for its whole life: it grows the set to what its load holds -//! and carves it, and its `destroy` releases every arena. Callers never -//! supply a workspace or a budget. +//! its workspace for its whole life: calibration maps its ring there, the +//! backend grows the set to what its load holds and carves it, and its +//! `destroy` releases every arena. Callers never supply a workspace, a +//! budget or a calibration result. //! //! The memory depends on the target: CUDA and oneAPI take page-backed arenas //! registered with PJRT (DmaMap), ROCm and Furiosa PJRT `pinned_host` @@ -231,15 +232,15 @@ const ArenaKind = union(enum) { } }; -/// Host arenas owned by one direct loader. Arenas are mapped one at a time +/// Host arenas owned by one direct loader. Calibration maps its ring /// (`allocate`); the loader then grows the set to what its load holds and /// cuts it into blocks, once (`carve`). Deinitialize after all transfers /// finish and before the platform. pub const Workspace = struct { /// Safety guard on the arenas' total host memory (pinned on the DMA - /// targets), not an allocation target: the load's pre-growth decides - /// what is actually mapped. Fixed, because no caller ever had a reason - /// to choose another value. + /// targets), not an allocation target: calibration and the load's + /// pre-growth decide what is actually mapped. Fixed, because no caller + /// ever had a reason to choose another value. pub const mapped_bytes_ceiling: usize = 16 * 1024 * 1024 * 1024; allocator: Allocator, @@ -405,8 +406,8 @@ test "Workspace arena ownership cleans up allocation failures" { _ = try workspace.allocate(64); _ = try workspace.allocate(128); - // The first step is already mapped; the second maps the last - // block. + // The first step is already mapped, as calibration's ring covers + // one device's reserve; the second maps the last block. const blocks = try workspace.carve(64, 3, 4); try std.testing.expectEqual(@as(usize, 4), blocks.len); try std.testing.expectEqual(@as(usize, 256), workspace.mappedBytes()); diff --git a/zml/io/loader.zig b/zml/io/loader.zig index a5755a8e..26833c51 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -125,7 +125,8 @@ pub const Loader = struct { pub const max_read_parallelism: usize = 128; /// Checks `opts` and returns at once. Warming the device allocators and - /// creating the backend run on a task of the loader's own, or here when + /// creating the backend (for the direct one: DMA calibration and the + /// pinned pre-growth) run on a task of the loader's own, or here when /// `io` has no concurrency; the first `start` waits for that task and /// returns its error. A read width out of range is `InvalidOptions`, /// logged. From e9746adbd042cec54dbed0802094844b736e0dff Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 13:49:37 +0100 Subject: [PATCH 079/103] zml/io: add direct backend metrics and debug diagnostics The direct backend now counts what an A/B run needs to explain a load time, in atomic counters the hot paths bump: reads and their bytes and time, the time workers wait for a lifecycle credit, how long a request holds its credit in the DMA stage, the time to create a tensor's buffers and transfer managers, DMA submissions with their submit and completion latency, and why each device pump stopped (empty queue or full budget). At debug level on the zml/io/load scope, each awaited batch logs a 'batch completed' line (when it was published, sealed and done, from the loader's creation) and a 'batch planning' line (plans, planning time, jobs, transfers, planned DMA submissions, coalescing ratio, average read size), which is only computed when it would be logged. destroy logs the 'loader summary', 'loader waits' and 'loader DMA' lines, with the credits' high water and the pinned bytes mapped. Nothing reads the counters but these logs. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 179 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 176 insertions(+), 3 deletions(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 574c1df5..26d291ea 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -89,6 +89,7 @@ pub const Loader = struct { /// and is never written after `create`. blocks: []const host_memory.Block, scheduler: Scheduler, + metrics: Metrics = .{}, /// One permit per source read in flight (`readSlot`), `read_width` of /// them. The width never changes: a source that answers 429 (or 503 on /// the object stores) is held one layer down, where the VFS holds every @@ -116,6 +117,8 @@ pub const Loader = struct { /// open until `destroy`. Keyed by the slot's own copy of the path: the /// registry that named it may go once the batch retired. source_slots: std.StringHashMapUnmanaged(*SourceSlot) = .empty, + /// What the debug lines time from (`logBatch`, `logSummary`). + created_at: std.Io.Timestamp, /// Submissions so far; the next batch's sequence number. batch_count: usize = 0, /// The block size and the request size, for the whole load. @@ -203,6 +206,7 @@ pub const Loader = struct { .credits = credits, .pumps = pumps, .dma_budget_bytes = dma_calibration.depth * block_size, + .created_at = .now(io, .awake), .plan_config = .{ .block_size = block_size, .request_size = request_size, @@ -251,11 +255,14 @@ pub const Loader = struct { // Fail the loader, seal and await what was published of this // batch, and return the loader's first error. self.recordError(err, "plan batch {d}, published_plans={d}", .{ batch.sequence, batch.plans.items.len }); + batch.sealed_at = .now(self.io, .awake); self.batch_count += 1; batch.releaseUnits(1); try self.awaitBatch(batch); unreachable; }; + // Stamped before the sentinel goes, so before the batch can complete. + batch.sealed_at = .now(self.io, .awake); self.batch_count += 1; // Every plan is visible: drop the publish sentinel. A batch without // jobs completes right here. @@ -279,6 +286,7 @@ pub const Loader = struct { /// panics on. pub fn awaitBatch(self: *Loader, batch: *Batch) !void { batch.done.waitUncancelable(self.io); + const done_at: std.Io.Timestamp = .now(self.io, .awake); // Every request of this batch has completed: no worker or callback // touches its managers or contexts any more. const load_error = self.first_error.get(); @@ -295,6 +303,7 @@ pub const Loader = struct { // Each destination completed its progress item. if (load_error == null) for (batch.destinations) |*destination| std.debug.assert(destination.unread_bytes.load(.acquire) == 0); Dma.retireBatch(self, batch); + self.logBatch(batch, done_at, load_error == null); self.destroyBatch(batch); if (load_error) |err| return err; } @@ -310,6 +319,7 @@ pub const Loader = struct { /// flight, so stopping the workers is a plain shutdown. pub fn destroy(self: *Loader) void { self.stopWorkers(); + self.logSummary(); var slots = self.source_slots.valueIterator(); while (slots.next()) |slot| { slot.*.deinit(self.io); @@ -421,11 +431,14 @@ pub const Loader = struct { /// waits for its slot. Null, with the credit given back, when nothing /// was left to claim. fn claimRequest(self: *Loader) ?*ReadRequest { + const credit_wait_started = awakeNs(self.io); const slot = self.credits.acquire(self.io); + const credit_wait_ns = awakeNs(self.io) -| credit_wait_started; const claim = self.scheduler.claim(self.io) orelse { self.credits.release(self.io, slot); return null; }; + _ = self.metrics.credit_wait_ns.fetchAdd(credit_wait_ns, .monotonic); return ReadRequest.init(self, claim, slot); } @@ -459,6 +472,88 @@ pub const Loader = struct { Dma.abortReady(self); } } + + /// Counts the plans' jobs, transfers and events only when the lines + /// would be logged. + fn logBatch(self: *Loader, batch: *const Batch, done_at: std.Io.Timestamp, successful: bool) void { + if (!std.log.logEnabled(.debug, .@"zml/io/load")) return; + const sealed_at = batch.sealed_at orelse self.created_at; + const published_at = batch.published_at orelse sealed_at; + var logical_bytes: usize = 0; + for (batch.destinations) |*destination| logical_bytes += destination.shape.byteSize(); + load_log.debug("batch completed: batch={d}, successful={}, logical_bytes={Bi:.2}, published=+{d:.3}s, sealed=+{d:.3}s, done=+{d:.3}s, elapsed={d:.3}s, source_width={d}, request_size={Bi:.2}", .{ + batch.sequence, + successful, + logical_bytes, + secondsBetween(self.created_at, published_at), + secondsBetween(self.created_at, sealed_at), + secondsBetween(self.created_at, done_at), + secondsBetween(published_at, done_at), + self.read_width, + self.plan_config.request_size, + }); + + var planning_ns: u64 = 0; + var source_bytes: u64 = 0; + var jobs: usize = 0; + var transfers: usize = 0; + var dma_submissions: usize = 0; + for (batch.plans.items) |plan| { + planning_ns += plan.planning_ns; + source_bytes += plan.source_bytes; + jobs += plan.jobs.len; + transfers += plan.transfers.len; + dma_submissions += plan.events.len; + } + const average_read_size = if (jobs == 0) 0 else source_bytes / jobs; + const coalescing_ratio = if (jobs == 0) + 0 + else + @as(f64, @floatFromInt(batch.parts.len)) / + @as(f64, @floatFromInt(jobs)); + load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ + batch.sequence, + batch.plans.items.len, + @as(f64, @floatFromInt(planning_ns)) / std.time.ns_per_s, + source_bytes, + jobs, + batch.parts.len, + transfers, + dma_submissions, + coalescing_ratio, + average_read_size, + }); + } + + fn logSummary(self: *Loader) void { + const reads = self.metrics.read_operations.load(.acquire); + load_log.debug("loader summary: batches={d}, successful={}, read_bytes={Bi:.2}, elapsed={d:.3}s, reads={d}, source_width={d}, request_size={Bi:.2}, credits_high_water={d} of {d}, pinned_mapped={Bi:.2}", .{ + self.batch_count, + self.first_error.get() == null, + self.metrics.read_bytes.load(.acquire), + secondsBetween(self.created_at, .now(self.io, .awake)), + reads, + self.read_width, + self.plan_config.request_size, + self.credits.high_water, + self.credits.count(), + self.workspace.mappedBytes(), + }); + load_log.debug("loader waits: credit_wait_ms_per_read={d:.3}, read_ms_per_read={d:.3}, dma_stage_ms_per_read={d:.3}, tensor_init_ms_per_read={d:.3}", .{ + millisecondsPer(self.metrics.credit_wait_ns.load(.acquire), reads), + millisecondsPer(self.metrics.read_ns.load(.acquire), reads), + millisecondsPer(self.metrics.dma_stage_ns.load(.acquire), reads), + millisecondsPer(self.metrics.tensor_init_ns.load(.acquire), reads), + }); + const submissions = self.metrics.dma_submissions.load(.acquire); + load_log.debug("loader DMA: dma_submissions={d}, dma_submit_us_per_piece={d:.2}, dma_piece_latency_ms={d:.3}, pump_stops_empty={d}, pump_stops_full={d}", .{ + submissions, + millisecondsPer(self.metrics.dma_submit_ns.load(.acquire), submissions) * 1000, + millisecondsPer(self.metrics.dma_piece_ns.load(.acquire), submissions), + self.metrics.pump_stops_empty.load(.acquire), + self.metrics.pump_stops_full.load(.acquire), + }); + } }; /// One submission: its plans (one per source file, published as each file @@ -488,6 +583,10 @@ pub const Batch = struct { done: std.Io.Event = .unset, /// Submission number within the loader, for log correlation. sequence: usize, + /// The first publish; null for a submission without plans. + published_at: ?std.Io.Timestamp = null, + /// Once every plan is published, before the publish sentinel goes. + sealed_at: ?std.Io.Timestamp = null, /// An open batch holding its publish sentinel and nothing else. The /// sentinel keeps the batch from completing before its last plan is @@ -664,6 +763,8 @@ const Plan = struct { events_used: std.atomic.Value(usize) = .init(0), /// The tensor bytes the jobs read. source_bytes: u64, + /// How long the plan took to cut. + planning_ns: u64 = 0, /// Next job to claim; owned by the scheduler mutex. cursor: usize = 0, @@ -707,8 +808,11 @@ const Planner = struct { var file_start: usize = 0; while (file_start < order.len) { const file_end = fileGroupEnd(parts, order, file_start); + const planning_started: std.Io.Timestamp = .now(io, .awake); const plan = try preparePlan(scheduler.allocator, batch.arena.allocator(), parts, order[file_start..file_end], config); + plan.planning_ns = @intCast(@max(planning_started.untilNow(io, .awake).nanoseconds, 0)); try scheduler.publish(io, batch, plan); + if (batch.published_at == null) batch.published_at = .now(io, .awake); file_start = file_end; } } @@ -1128,6 +1232,8 @@ const ReadRequest = struct { /// The scheduling sentinel and one reference per DMA piece queued; the /// last release gives the credit back and finishes the job. pending: std.atomic.Value(usize) = .init(1), + /// Awake-clock nanoseconds of the enqueue; 0 until then. + enqueued_ns: u64 = 0, /// The slot of a job that was never claimed: nothing pending, so the /// retirement checks hold. @@ -1164,10 +1270,12 @@ const ReadRequest = struct { const destination = transfer.destination; if (destination == previous) continue; previous = destination; + const init_started = awakeNs(loader.io); _ = destination.ensureState(loader.allocator, loader.io, loader.platform) catch |err| { loader.recordError(err, "initialize tensor {s} from {s}, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); return err; }; + _ = loader.metrics.tensor_init_ns.fetchAdd(awakeNs(loader.io) -| init_started, .monotonic); } const blocks = loader.credits.slotBlocks(self.slot)[0..job.blockLen(block_size)]; switch (job.kind) { @@ -1200,6 +1308,9 @@ const ReadRequest = struct { // that owns this request, so nothing is touched after it. const loader = self.loader; const batch = self.plan.batch; + if (self.enqueued_ns != 0) { + _ = loader.metrics.dma_stage_ns.fetchAdd(awakeNs(loader.io) -| self.enqueued_ns, .monotonic); + } loader.credits.release(loader.io, self.slot); batch.releaseUnits(1); } @@ -1247,10 +1358,15 @@ fn readSlot(loader: *Loader, slot: *SourceSlot, iovecs: [][]u8, file_offset: u64 loader.read_permits.waitUncancelable(io); defer loader.read_permits.post(io); if (loader.first_error.get()) |err| return err; - _ = readFilePositionalAllV(io, file, iovecs, file_offset, minimum_len) catch |err| { + const read_started = awakeNs(io); + const read_result = readFilePositionalAllV(io, file, iovecs, file_offset, minimum_len); + _ = loader.metrics.read_ns.fetchAdd(awakeNs(io) -| read_started, .monotonic); + const bytes_read = read_result catch |err| { loader.recordError(err, "read {s} at {d}, {d} bytes at least", .{ slot.uri, file_offset, minimum_len }); return err; }; + _ = loader.metrics.read_operations.fetchAdd(1, .monotonic); + _ = loader.metrics.read_bytes.fetchAdd(@intCast(bytes_read), .monotonic); } /// The bytes from offset `at` of `blocks` taken end to end, `block_size` @@ -1340,6 +1456,8 @@ const Dma = struct { device_index: usize, /// Bytes this submission holds against the device's in-flight budget. len: usize, + /// Awake-clock nanoseconds of the submission. + submitted_ns: u64 = 0, /// Link of the device pump's `retired`; owned by that pump's mutex. next_retired: ?*EventContext = null, @@ -1414,6 +1532,7 @@ const Dma = struct { counts[tensor.targets[writer_index].device_index] += 1; } } + const enqueued_ns = awakeNs(loader.io); // Every pump is held while the request's pieces land, and every // queue is reserved before any changes, so the request is either // fully queued or not queued at all. @@ -1431,6 +1550,7 @@ const Dma = struct { // Every piece holds the request, and with it the slot, until its // callback. _ = request.pending.fetchAdd(pieces, .acq_rel); + request.enqueued_ns = enqueued_ns; for (transfers) |transfer| { // The planner cut the transfers to the job's blocks. const data = blockPiece(blocks, block_size, transfer.landing, transfer.len); @@ -1478,11 +1598,15 @@ const Dma = struct { var selected: ?Piece = null; device_pump.mutex.lockUncancelable(loader.io); device_pump.destroyRetired(); - if (loader.first_error.get() == null and device_pump.hasRoom(loader.dma_budget_bytes)) { - if (device_pump.queue.popFront()) |piece| { + if (loader.first_error.get() == null) { + if (!device_pump.hasRoom(loader.dma_budget_bytes)) { + _ = loader.metrics.pump_stops_full.fetchAdd(1, .monotonic); + } else if (device_pump.queue.popFront()) |piece| { device_pump.active_bytes += piece.data.len; device_pump.active_pieces += 1; selected = piece; + } else { + _ = loader.metrics.pump_stops_empty.fetchAdd(1, .monotonic); } } if (selected == null) device_pump.pumping = false; @@ -1504,6 +1628,7 @@ const Dma = struct { const api = loader.platform.pjrt_api; const target = piece.target; const is_last = target.nextIsLast(piece.data.len); + const submit_started = awakeNs(loader.io); const event = try target.manager.transferData( api, 0, @@ -1524,8 +1649,10 @@ const Dma = struct { .pjrt_event = event, .device_index = target.device_index, .len = piece.data.len, + .submitted_ns = submit_started, }; + _ = loader.metrics.dma_submissions.fetchAdd(1, .monotonic); event.onReady(api, EventContext, struct { fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { const request = ctx_.request; @@ -1533,6 +1660,7 @@ const Dma = struct { const api_ = loader_.platform.pjrt_api; const device_index = ctx_.device_index; const len = ctx_.len; + _ = loader_.metrics.dma_piece_ns.fetchAdd(awakeNs(loader_.io) -| ctx_.submitted_ns, .monotonic); // The shipped plugins resolve this event without an error // whatever happened to the copy (the C API wrapper sets the // promise with an OK status); kept for a plugin that reports. @@ -1548,6 +1676,7 @@ const Dma = struct { ctx.destroyEvent(); return err; }; + _ = loader.metrics.dma_submit_ns.fetchAdd(awakeNs(loader.io) -| submit_started, .monotonic); } /// Ends a submitted piece: frees its room, which may pump on this thread; @@ -1758,6 +1887,35 @@ const Credits = struct { } }; +/// The debug summary's counters (`Loader.logSummary`): read only by logs; +/// nothing here decides. +const Metrics = struct { + read_operations: std.atomic.Value(u64) = .init(0), + read_bytes: std.atomic.Value(u64) = .init(0), + dma_submissions: std.atomic.Value(u64) = .init(0), + /// Time workers spend waiting for a lifecycle credit and inside the + /// source read, summed over requests: waits above the read time mean the + /// load is DMA-completion bound. + credit_wait_ns: std.atomic.Value(u64) = .init(0), + read_ns: std.atomic.Value(u64) = .init(0), + /// From a request's enqueue to its last DMA callback: how long the DMA + /// stage holds a lifecycle credit and pinned blocks per request. + dma_stage_ns: std.atomic.Value(u64) = .init(0), + /// Inside `submitTransfer` (the PJRT call and the callback registration) + /// and from a submission to its ready callback, summed over submissions. + dma_submit_ns: std.atomic.Value(u64) = .init(0), + dma_piece_ns: std.atomic.Value(u64) = .init(0), + /// Why a device pump stopped: nothing queued, or no room under the + /// device's budget. A pump that mostly stops empty is starved by the + /// reads; one that mostly stops full is waiting on the engine. + pump_stops_empty: std.atomic.Value(u64) = .init(0), + pump_stops_full: std.atomic.Value(u64) = .init(0), + /// Inside `ensureState` for a claimed job's destinations: the first + /// worker to touch a tensor creates its PJRT buffers and transfer + /// managers there, and the other workers of the same tensor wait for it. + tensor_init_ns: std.atomic.Value(u64) = .init(0), +}; + /// A value initialized at most once by whichever task touches it first, /// with the initializer that task passes. Concurrent callers wait on the /// event; a failed initialization keeps its error code and re-materializes @@ -1809,6 +1967,20 @@ fn LazyOnce(comptime T: type) type { }; } +fn secondsBetween(from: std.Io.Timestamp, to: std.Io.Timestamp) f64 { + return @as(f64, @floatFromInt(from.durationTo(to).nanoseconds)) / std.time.ns_per_s; +} + +/// Mean milliseconds per unit of a summed duration (zero units: the total). +fn millisecondsPer(total_ns: u64, units: u64) f64 { + return @as(f64, @floatFromInt(total_ns)) / std.time.ns_per_ms / @as(f64, @floatFromInt(@max(units, 1))); +} + +fn awakeNs(io: std.Io) u64 { + const now: std.Io.Timestamp = .now(io, .awake); + return @intCast(@max(now.nanoseconds, 1)); +} + /// Writes `contents` to a new file `name` in `dir` and returns its real /// path, in `path_buffer`. fn writeTestFile(dir: std.Io.Dir, io: std.Io, name: []const u8, contents: []const u8, path_buffer: []u8) ![]const u8 { @@ -2271,6 +2443,7 @@ const TestLoader = struct { .credits = try .init(allocator, blocks[1..], 1), .pumps = &self.pumps, .dma_budget_bytes = 64, + .created_at = .now(std.testing.io, .awake), .plan_config = .{ .block_size = 64, .request_size = 64 }, }; } From f926de23c03034008ab5640169f8ff775346f5ea Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 14:29:06 +0100 Subject: [PATCH 080/103] zml/io: expose the new loader as zml.io.next and add load_check zml.io.next exports the new Loader and its TensorStore next to the zml.io.Loader and zml.io.TensorStore that models still use. It is unstable, and goes away once the new loader replaces the old one. zml.testing's loader helpers name it instead of importing its files by path, and the docs that pointed at the unexported store now name it. //examples/io:load_check is the tool the new loader's backends are validated with on each host. It binds every tensor of a checkpoint, local or remote, replicated, split along its first axis, or as tensor parallelism splits o_proj and down_proj, loads them through zml.io.next, and with ZML_LOAD_CHECK reads them back and compares every replica with the checkpoint's bytes. The loader logs its rate and its stats on the zml/io/load scope. The knobs, listed in the file's header, also set a GPU memory pool fraction and the CPU device count. The examples/io demo keeps the old loader. Co-Authored-By: Claude Opus 5.5 --- examples/io/BUILD.bazel | 7 + examples/io/load_check.zig | 261 +++++++++++++++++++++++++++++++++++++ zml/Compiler.zig | 2 +- zml/io.zig | 12 +- zml/io/TensorStore.zig | 12 +- zml/io/direct_loader.zig | 6 +- zml/testing.zig | 12 +- 7 files changed, 292 insertions(+), 20 deletions(-) create mode 100644 examples/io/load_check.zig diff --git a/examples/io/BUILD.bazel b/examples/io/BUILD.bazel index d471f74d..785ab603 100644 --- a/examples/io/BUILD.bazel +++ b/examples/io/BUILD.bazel @@ -9,3 +9,10 @@ zig_binary( deps = ["//zml"], visibility = ["//visibility:public"], ) + +# Tooling: the new loader's validation harness. Its header lists its knobs. +zig_binary( + name = "load_check", + main = "load_check.zig", + deps = ["//zml"], +) diff --git a/examples/io/load_check.zig b/examples/io/load_check.zig new file mode 100644 index 00000000..d22ff3f1 --- /dev/null +++ b/examples/io/load_check.zig @@ -0,0 +1,261 @@ +//! Loads every tensor of a checkpoint through `zml.io.next` and checks the +//! bytes the devices hold against the checkpoint's. Tooling, not an +//! example: the harness the new loader's backends are validated with on +//! each host. +//! +//! bazel run //examples/io:load_check -- [replicated|sharded|tp] +//! +//! The checkpoint is a model directory, a .safetensors file or a +//! .safetensors.index.json, local or remote (hf://, s3://, gs://, https://, +//! with the credentials and proxies of the environment). Each tensor is +//! placed on the `tp` mesh, all the platform's devices on its `model` axis, +//! by the mode: +//! - `replicated`: whole on every device; +//! - `sharded`, the default: split along its first axis, which the devices +//! must divide, whatever the model (DeepSeek-V4.x cannot bind on 2 or 4 +//! devices); +//! - `tp`: the input axis of `o_proj` and `down_proj` weights, as tensor +//! parallelism shards them (column-sharded: every device's bytes are +//! strided in the checkpoint), the first axis of the others. +//! A scalar is replicated in every mode. +//! +//! The loader logs the bytes it loaded, its time and rate at info, and its +//! stats at debug, on the `zml/io/load` scope; `zml/vfs` logs the remote +//! read totals. This tool enables both. +//! +//! Knobs, from the environment: +//! - `ZML_LOAD_CHECK=n`: once loaded, read every n-th tensor and the largest +//! one back to host and compare them with the checkpoint's bytes, every +//! replica of a tensor each device holds whole. 1 checks every tensor; +//! unset or 0, none. A difference is `LoadContentMismatch`, logged. +//! - `ZML_GPU_MEMORY_FRACTION=f`: a preallocated BFC pool of that fraction +//! of each device's memory on CUDA, ROCm and oneAPI, instead of the +//! platform's allocator defaults, for a run that observes the loader's +//! admission on a device with room. +//! - `ZML_CPU_DEVICE_COUNT=n`: the CPU platform's devices (4 by default). +const std = @import("std"); + +const zml = @import("zml"); + +const log = std.log.scoped(.load_check); + +pub const std_options: std.Options = .{ + .log_level = .info, + .log_scope_levels = &.{ + .{ .scope = .@"zml/io/load", .level = .debug }, + .{ .scope = .@"zml/vfs", .level = .debug }, + }, +}; + +const Mode = enum { replicated, sharded, tp }; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + var args = init.minimal.args.iterate(); + _ = args.next(); // skip program name + const path = args.next() orelse { + log.err("usage: load_check [replicated|sharded|tp]", .{}); + return error.MissingPath; + }; + const mode = std.meta.stringToEnum(Mode, args.next() orelse "sharded") orelse { + log.err("the mode must be replicated, sharded or tp", .{}); + return error.InvalidArgument; + }; + const check_stride = try envNumber(usize, init.environ_map, "ZML_LOAD_CHECK") orelse 0; + + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + + var remotes: zml.io.VFS.Remotes = undefined; + try remotes.init(allocator, &vfs, init.environ_map); + defer remotes.deinit(); + + const io = vfs.io(); + + const platform: *zml.Platform = try .auto(allocator, io, try platformOptions(init.environ_map)); + defer platform.deinit(allocator, io); + + // Returns at once: the loader sets its backend up while the + // checkpoint's headers are read and its tensors bound. + var loader: zml.io.next.Loader = try .init(allocator, io, platform, .{}); + defer loader.deinit(); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + + const tensor_parallel: *const zml.Mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); + const model_axis: u3 = @intCast(tensor_parallel.resolveLogicalAxis("model").?); + const rows: zml.Sharding.Partitioning = .init(&.{.sharded(model_axis)}); + const columns: zml.Sharding.Partitioning = .init(&.{ .replicated, .sharded(model_axis) }); + + var store: zml.io.next.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); + defer store.deinit(); + + const Model = struct { tensors: []zml.Tensor }; + // The checkpoint tensor each one is bound to, at the same index. + const sources = store.registry.tensors.values(); + const model: Model = .{ .tensors = try allocator.alloc(zml.Tensor, sources.len) }; + defer allocator.free(model.tensors); + const view = store.view(); + for (model.tensors, store.registry.tensors.keys(), sources) |*tensor, key, source| { + const rank = source.shape.rank(); + const column_sharded = rank == 2 and + (std.mem.endsWith(u8, key, "o_proj.weight") or std.mem.endsWith(u8, key, "down_proj.weight")); + tensor.* = if (rank == 0 or mode == .replicated) + view.createReplicatedTensor(key, null) + else if (mode == .tp and column_sharded) + view.createTensor(key, null, .tp, columns) + else + view.createTensor(key, null, .tp, rows); + } + + var progress = std.Progress.start(io, .{ .root_name = "load_check", .disable_printing = true }); + defer progress.end(); + + var buffers = try zml.mem.bufferize(allocator, &model); + defer zml.mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &store, progress); + try loader.wait(); + + if (check_stride != 0) try checkLoaded(allocator, io, sources, buffers.tensors, check_stride); +} + +/// The platform settings of the environment, which are not loader knobs. +fn platformOptions(environ_map: *const std.process.Environ.Map) !zml.platform.CreateOptions { + var options: zml.platform.CreateOptions = .{}; + if (try envNumber(u32, environ_map, "ZML_CPU_DEVICE_COUNT")) |device_count| { + options.cpu.device_count = device_count; + } + // Unset keeps the platform's allocator defaults (VMM on CUDA and ROCm); + // set, every XLA GPU target gets a preallocated BFC pool of that + // fraction, whose `bytes_limit` admission measures against. + if (try envNumber(f32, environ_map, "ZML_GPU_MEMORY_FRACTION")) |memory_fraction| { + const pool: zml.platform.CreateOptions.XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }; + options.cuda = pool; + options.rocm = pool; + options.oneapi = pool; + } + return options; +} + +/// Reads every `stride`-th of `buffers` and the largest eligible one back to +/// host and compares the bytes with the checkpoint tensor each was bound +/// to, `sources` at the same index, so a buffer that reported ready before +/// every piece landed is caught. A buffer every device holds whole is +/// compared on every replica; a partitioned one is assembled with +/// `toSliceAlloc`, which keeps the last replica of a region that several +/// devices hold. Each source file is opened once (an `hf://` open is a HEAD +/// plus a redirect). Skipped: sub-byte tensors partitioned over several +/// devices (`toSliceAlloc` places their shards by element stride). +fn checkLoaded( + allocator: std.mem.Allocator, + io: std.Io, + sources: []const zml.safetensors.Tensor, + buffers: []const zml.Buffer, + stride: usize, +) !void { + std.debug.assert(sources.len == buffers.len); + const Eligibility = struct { + fn checkable(source: zml.safetensors.Tensor, buffer: zml.Buffer) bool { + const partitioned = buffer.byteSize() / buffer.numShards() != source.byteSize(); + return !partitioned or source.shape.dtype().bitSizeOf() >= 8; + } + }; + var largest: ?usize = null; + for (sources, buffers, 0..) |source, buffer, i| { + if (!Eligibility.checkable(source, buffer)) continue; + if (largest == null or source.byteSize() > sources[largest.?].byteSize()) largest = i; + } + var files: std.StringHashMapUnmanaged(std.Io.File) = .empty; + defer { + var it = files.valueIterator(); + while (it.next()) |file| file.close(io); + files.deinit(allocator); + } + var expected: []u8 = &.{}; + defer allocator.free(expected); + var replica: []u8 = &.{}; + defer allocator.free(replica); + const started: std.Io.Timestamp = .now(io, .awake); + var checked: usize = 0; + var skipped: usize = 0; + var checked_bytes: usize = 0; + for (sources, buffers, 0..) |source, buffer, i| { + if (i % stride != 0 and i != largest) continue; + if (!Eligibility.checkable(source, buffer)) { + skipped += 1; + continue; + } + const file = files.get(source.file_uri) orelse blk: { + const opened = try std.Io.Dir.openFile(.cwd(), io, source.file_uri, .{ .mode = .read_only }); + errdefer opened.close(io); + try files.put(allocator, source.file_uri, opened); + break :blk opened; + }; + const source_bytes: usize = @intCast(source.byteSize()); + if (expected.len < source_bytes) { + allocator.free(expected); + expected = &.{}; + expected = try allocator.alloc(u8, source_bytes); + } + try source.readRange(io, file, expected[0..source_bytes], 0); + + if (buffer.byteSize() / buffer.numShards() == source_bytes) { + // Every shard holds the whole tensor: compare each replica. + if (replica.len < source_bytes) { + allocator.free(replica); + replica = &.{}; + replica = try allocator.alloc(u8, source_bytes); + } + var shards = buffer.shards(); + var shard_index: usize = 0; + while (shards.next()) |shard| : (shard_index += 1) { + try shard.toHost(io, replica[0..source_bytes]); + try expectSameBytes(source.name, shard_index, expected[0..source_bytes], replica[0..source_bytes]); + checked_bytes += source_bytes; + } + } else { + const slice = try buffer.toSliceAlloc(allocator, io); + defer slice.free(allocator); + try expectSameBytes(source.name, null, expected[0..source_bytes], slice.constData()); + checked_bytes += source_bytes; + } + checked += 1; + } + log.info("load check: ok tensors_checked={d} of {d} skipped={d} bytes={Bi:.2} elapsed={f}", .{ + checked, + sources.len, + skipped, + checked_bytes, + started.untilNow(io, .awake), + }); +} + +fn expectSameBytes(name: []const u8, replica: ?usize, expected: []const u8, actual: []const u8) !void { + if (actual.len == expected.len and std.mem.eql(u8, expected, actual)) return; + const first_bad = std.mem.indexOfDiff(u8, expected, actual) orelse @min(expected.len, actual.len); + log.err("load check: mismatch tensor={s} replica={?d} expected_bytes={d} actual_bytes={d} first_difference={d}", .{ + name, + replica, + expected.len, + actual.len, + first_bad, + }); + return error.LoadContentMismatch; +} + +/// The environment variable `name` parsed as a `T`, null when unset. A value +/// that does not parse is `InvalidArgument`, logged. +fn envNumber(comptime T: type, environ_map: *const std.process.Environ.Map, name: []const u8) !?T { + const text = environ_map.get(name) orelse return null; + const value = switch (@typeInfo(T)) { + .int => std.fmt.parseInt(T, text, 10), + .float => std.fmt.parseFloat(T, text), + else => @compileError("envNumber: " ++ @typeName(T)), + }; + return value catch { + log.err("{s} must be a number, not '{s}'", .{ name, text }); + return error.InvalidArgument; + }; +} diff --git a/zml/Compiler.zig b/zml/Compiler.zig index eebf0be5..d3955786 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -84,7 +84,7 @@ pub const Options = struct { /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. /// XLA then counts them against device memory, leaves no budget for temporary buffers and /// rematerializes aggressively. Callers set this to the per-device size of such buffers to - /// compensate: `TensorStore.View.hostMappedBytes` for those the loader delivers + /// compensate: `zml.io.next.TensorStore.View.hostMappedBytes` for those the loader delivers /// `.host_mapped`. Remove once XLA properly supports host-pinned inputs. extra_device_memory_bytes: u64 = 0, // Debugging options diff --git a/zml/io.zig b/zml/io.zig index f8d10103..3a633c65 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -24,8 +24,16 @@ const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/io"); -// The new loader's files under io/, which zml does not export: only their -// tests reach them, from here. +/// A preview of the loader that replaces `Loader` and `TensorStore` below: +/// the new `Loader` (io/loader.zig) and its store (io/TensorStore.zig). +/// Unstable: their API may still change until they replace the old ones. +pub const next = struct { + pub const Loader = @import("io/loader.zig").Loader; + pub const TensorStore = @import("io/TensorStore.zig"); +}; + +// Every file of the new loader under io/, for its tests: `next` exports +// only some of them. test { _ = @import("io/ShardGrid.zig"); _ = @import("io/Source.zig"); diff --git a/zml/io/TensorStore.zig b/zml/io/TensorStore.zig index 4d6043c5..78567334 100644 --- a/zml/io/TensorStore.zig +++ b/zml/io/TensorStore.zig @@ -1,8 +1,8 @@ -//! The new loader's store: it associates model tensors with checkpoint -//! sources and prefixed views, over the entries it takes from a checkpoint -//! registry. zml does not export it: models and the loader they use build -//! `zml.io.TensorStore` (io.zig), which borrows its registry, so production -//! changes go there and the new loader's here. +//! Associates model tensors with checkpoint sources and prefixed views, +//! over the entries it takes from a checkpoint registry. zml exports it as +//! `zml.io.next.TensorStore`, the store `zml.io.next.Loader` loads from; +//! `zml.io.TensorStore` (io.zig) is the old loader's, which borrows its +//! registry. //! //! The store owns those entries, the bindings, and every source and program //! value (its arena). Views and sources point to it: once `view` was @@ -285,7 +285,7 @@ pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mes } std.debug.panic( \\Found no meshes named {s} in TensorStore. - \\Try passing more meshes to `zml.io.TensorStore.init`. + \\Try passing more meshes to `zml.io.next.TensorStore.init`. \\Known meshes: {f} , .{ name_slice, stdx.fmt.slice(store.meshes) }); } diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 26d291ea..8a70252c 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -138,9 +138,9 @@ pub const Loader = struct { var workspace: host_memory.Workspace = try .init(allocator, io, platform); errdefer workspace.deinit(); - // At debug when canceled (`zml.io.Loader.deinit` cancels a set-up - // no `start` waited for) or out of memory (the workspace logs what - // it fails to map or pin). + // At debug when canceled (`zml.io.next.Loader.deinit` cancels a + // set-up no `start` waited for) or out of memory (the workspace logs + // what it fails to map or pin). const block_size = dma_calibration.calibrate(&workspace, platform, .{ .block_sizes = opts.dmaBlockSizes() }) catch |err| { load_log.failure(err, "calibrate DMA: target={t}", .{platform.target}); return err; diff --git a/zml/testing.zig b/zml/testing.zig index f6e1b142..55516494 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -7,10 +7,6 @@ const Platform = @import("platform.zig").Platform; const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; const zml = @import("zml.zig"); -/// The new loader and its store, which zml does not export. -const Loader = @import("io/loader.zig").Loader; -const LoaderStore = @import("io/TensorStore.zig"); -const LoadMemory = LoaderStore.Memory; const log = std.log.scoped(.@"zml/testing"); @@ -482,7 +478,7 @@ pub fn pattern(bytes: []u8, seed: u8) void { /// A store of the new loader over a copy of `registry`, for a test that /// builds several stores over one checkpoint: a store takes its registry's /// entries. -pub fn storeOf(allocator: std.mem.Allocator, registry: *const zml.safetensors.TensorRegistry, opts: LoaderStore.Options) !LoaderStore { +pub fn storeOf(allocator: std.mem.Allocator, registry: *const zml.safetensors.TensorRegistry, opts: zml.io.next.TensorStore.Options) !zml.io.next.TensorStore { var copy: zml.safetensors.TensorRegistry = .init(allocator); defer copy.deinit(); for (registry.tensors.values()) |tensor| try copy.registerTensor(tensor); @@ -498,11 +494,11 @@ pub fn load( io: std.Io, platform: *const Platform, model: anytype, - store: *const LoaderStore, + store: *const zml.io.next.TensorStore, ) !zml.Bufferized(@TypeOf(model.*)) { var buffers = try zml.mem.bufferize(allocator, model); errdefer zml.mem.deinitBufferized(allocator, &buffers); - var loader = try Loader.init(allocator, io, platform, .{}); + var loader = try zml.io.next.Loader.init(allocator, io, platform, .{}); defer loader.deinit(); try loader.start(model, &buffers, store, .none); try loader.wait(); @@ -515,7 +511,7 @@ pub fn load( /// A buffer without a mesh is replicated on the platform's devices. On /// CPU/neuron/metal every memory kind is the one memory space, so only /// CUDA/ROCm/oneAPI tell them apart. -pub fn expectShards(allocator: std.mem.Allocator, io: std.Io, buffer: *const zml.Buffer, global: []const u8, memory: LoadMemory) !void { +pub fn expectShards(allocator: std.mem.Allocator, io: std.Io, buffer: *const zml.Buffer, global: []const u8, memory: zml.io.next.TensorStore.Memory) !void { const platform = buffer._platform; const shape = platform.placedShape(buffer.shape()).packedShape(); try std.testing.expectEqual(shape.byteSize(), global.len); From 58251571638428ee41ad72cd88200f1adb5398d0 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 15:06:07 +0100 Subject: [PATCH 081/103] zml/io: size reads from VFS hints and read with direct I/O zml.io.next.Loader.Options gains the checkpoint, a DMA block and a direct-I/O policy, and its read width becomes optional. When io is a VFS, init asks the VFS backend serving the checkpoint for its read hints, which size the source requests, and takes a default width of 16 for a local checkpoint and 32 for a high-latency one. It checks the width, the hints, and every block calibration may select against the request size and the direct-I/O alignment, and refuses the rest as InvalidOptions, logged. The direct backend asks the VFS once per local file whether it reads the file directly. Such a file is read in whole alignment units: each job's read widens to aligned bounds around its range and its transfers address the widened read; a job covers at most a request less two units, so the widened read still fits its slot. A remote file is read exact without being opened by the planner, and so is every file of a plain io or of the off policy. load_check takes the read width, the DMA block and the direct-I/O policy from ZML_LOAD_READ_PARALLELISM, ZML_DMA_BENCH_BLOCK_MIB and ZML_DIRECT_IO, and passes the checkpoint to the loader. Co-Authored-By: Claude Opus 5.5 --- examples/io/load_check.zig | 19 +++- zml/io/backend.zig | 19 ++-- zml/io/direct_loader.zig | 205 ++++++++++++++++++++++++++++++++----- zml/io/dma_calibration.zig | 16 +-- zml/io/loader.zig | 130 +++++++++++++++++++---- 5 files changed, 324 insertions(+), 65 deletions(-) diff --git a/examples/io/load_check.zig b/examples/io/load_check.zig index d22ff3f1..5ef8e9e0 100644 --- a/examples/io/load_check.zig +++ b/examples/io/load_check.zig @@ -28,6 +28,13 @@ //! one back to host and compare them with the checkpoint's bytes, every //! replica of a tensor each device holds whole. 1 checks every tensor; //! unset or 0, none. A difference is `LoadContentMismatch`, logged. +//! - `ZML_LOAD_READ_PARALLELISM=n`: the loader's concurrent source reads +//! (`Loader.Options.read_parallelism`); unset, the checkpoint's default +//! width, 16 locally and 32 remotely. +//! - `ZML_DMA_BENCH_BLOCK_MIB=n`: the direct backend's DMA block, in MiB +//! (`Loader.Options.dma_block_size`); unset, calibration picks one. +//! - `ZML_DIRECT_IO=off|on|auto`: direct I/O for local checkpoint files +//! (`Loader.Options.direct_io`, `VFS.DirectIo`); `auto` when unset. //! - `ZML_GPU_MEMORY_FRACTION=f`: a preallocated BFC pool of that fraction //! of each device's memory on CUDA, ROCm and oneAPI, instead of the //! platform's allocator defaults, for a run that observes the loader's @@ -63,6 +70,11 @@ pub fn main(init: std.process.Init) !void { return error.InvalidArgument; }; const check_stride = try envNumber(usize, init.environ_map, "ZML_LOAD_CHECK") orelse 0; + const direct_io = std.meta.stringToEnum(zml.io.VFS.DirectIo, init.environ_map.get("ZML_DIRECT_IO") orelse "auto") orelse { + log.err("ZML_DIRECT_IO must be off, on or auto", .{}); + return error.InvalidArgument; + }; + const dma_block_mib = try envNumber(u32, init.environ_map, "ZML_DMA_BENCH_BLOCK_MIB"); var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); @@ -78,7 +90,12 @@ pub fn main(init: std.process.Init) !void { // Returns at once: the loader sets its backend up while the // checkpoint's headers are read and its tensors bound. - var loader: zml.io.next.Loader = try .init(allocator, io, platform, .{}); + var loader: zml.io.next.Loader = try .init(allocator, io, platform, .{ + .checkpoint = path, + .read_parallelism = try envNumber(usize, init.environ_map, "ZML_LOAD_READ_PARALLELISM"), + .dma_block_size = if (dma_block_mib) |mib| @as(usize, mib) * zml.MiB else null, + .direct_io = direct_io, + }); defer loader.deinit(); var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); diff --git a/zml/io/backend.zig b/zml/io/backend.zig index 5f005a21..44dfdbfb 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -25,7 +25,8 @@ pub const max_read_request_size: usize = 32 * 1024 * 1024; /// a nonzero multiple of `alignment` (`readAlignment`: reads land in /// blocks), and the request is at most `max_read_request_size` and, with an /// alignment, longer than two units, so a read widened by a unit at each -/// end fits. +/// end fits. `zml.io.next.Loader.init` refuses options for which a block +/// calibration may select has none. pub fn requestSize(read_chunk_size: usize, block_size: usize, alignment: usize) ?usize { if (block_size == 0 or block_size % @max(alignment, 1) != 0) return null; // Bounded before the rounding, which then cannot overflow. @@ -37,24 +38,22 @@ pub fn requestSize(read_chunk_size: usize, block_size: usize, alignment: usize) return request; } -/// What a backend widens the reads of a file read directly to: the VFS's -/// alignment when `io` is a VFS and `direct_io` is not `off`, else 0, every -/// read exact. The VFS decides per local file whether it is read directly -/// (`VFS.useDirectIo`). +/// What the direct backend widens the reads of a file read directly to: +/// the VFS's alignment when `io` is a VFS and `direct_io` is not `off`, else +/// 0, every read exact. The VFS decides per local file whether it is read +/// directly (`VFS.useDirectIo`). pub fn readAlignment(io: std.Io, direct_io: VFS.DirectIo) usize { if (direct_io == .off or VFS.fromIo(io) == null) return 0; return VFS.direct_io_alignment; } -/// The loader's options as its front end checked them, for the backends. +/// `zml.io.next.Loader.Options` as `zml.io.next.Loader.init` checked them, +/// the checkpoint resolved to its VFS backend's read hints and the read +/// width to its default. See `zml.io.next.Loader.Options` for each field. pub const Options = struct { - /// Source reads in flight at once. read_parallelism: usize, - /// The checkpoint's VFS backend's read hints. read_hints: VFS.ReadHints = .local, - /// The DMA block, else one calibration picks. dma_block_size: ?usize = null, - /// Whether local files are read directly (`VFS.useDirectIo`). direct_io: VFS.DirectIo = .auto, /// The blocks calibration may select: `dma_block_size`, else diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 8a70252c..dff6ebd7 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -33,6 +33,14 @@ //! until every job completes; a failure retires the unclaimed jobs and //! drains what is in flight before the batch goes. //! +//! Direct I/O. A local file the VFS reads past the page cache +//! (`VFS.useDirectIo`, asked once per file: `Planner.Reads`) is read in +//! whole alignment units: each job's read widens to aligned bounds around +//! its range, into blocks that are whole units, and its transfers address +//! the widened read. A job's range is at most a request less two units, so +//! its widened read still fits its credit's slot. Every other file is read +//! exact. +//! //! Destinations. A part's source bytes map to its destination's logical //! offsets, which the destination's `ShardGrid` places on the devices by //! arithmetic. Pieces of one destination merge across parts when block, @@ -49,6 +57,7 @@ const std = @import("std"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); +const VFS = @import("vfs"); const Buffer = @import("../buffer.zig").Buffer; const pjrtx = @import("../pjrtx.zig"); @@ -113,9 +122,10 @@ pub const Loader = struct { /// Per-device in-flight budget: `dma_calibration.depth` blocks, in bytes. dma_budget_bytes: usize, worker_group: std.Io.Group = .init, - /// One per source path. Its file opens at its first read and stays - /// open until `destroy`. Keyed by the slot's own copy of the path: the - /// registry that named it may go once the batch retired. + /// One per source path. Its file opens at first use (during planning + /// when a VFS decides direct IO for a local file, otherwise at the first + /// read) and stays open until `destroy`. Keyed by the slot's own copy of + /// the path: the registry that named it may go once the batch retired. source_slots: std.StringHashMapUnmanaged(*SourceSlot) = .empty, /// What the debug lines time from (`logBatch`, `logSummary`). created_at: std.Io.Timestamp, @@ -123,6 +133,8 @@ pub const Loader = struct { batch_count: usize = 0, /// The block size and the request size, for the whole load. plan_config: Planner.Config, + /// How each file is read: exact, or aligned for direct I/O. + reads: Planner.Reads, pub fn create( allocator: std.mem.Allocator, @@ -145,8 +157,8 @@ pub const Loader = struct { load_log.failure(err, "calibrate DMA: target={t}", .{platform.target}); return err; }; - // Every read is exact. - const request_size = backend.requestSize(opts.read_hints.read_chunk_size, block_size, 0).?; + const reads: Planner.Reads = .init(io, opts.direct_io); + const request_size = backend.requestSize(opts.read_hints.read_chunk_size, block_size, reads.alignment).?; const blocks_per_request = request_size / block_size; // The DMA stage of every device, kept mapped beside the source // working set; at least one maximal source request, so a single @@ -211,6 +223,7 @@ pub const Loader = struct { .block_size = block_size, .request_size = request_size, }, + .reads = reads, }; std.debug.assert(self.dma_budget_bytes > 0); errdefer self.scheduler.deinit(); @@ -220,9 +233,11 @@ pub const Loader = struct { load_log.err("start {d} source workers: {s}", .{ workers, @errorName(err) }); return err; }; - load_log.debug("live loader ready: target={s}, request_size={Bi:.2}, dma_block_size={Bi:.2}, dma_budget_per_device={Bi:.2}, source_width={d}, lifecycle_credits={d}, workers={d}, retained={Bi:.2}", .{ + load_log.debug("live loader ready: target={s}, request_size={Bi:.2}, direct_io={t}, source_alignment={d}, dma_block_size={Bi:.2}, dma_budget_per_device={Bi:.2}, source_width={d}, lifecycle_credits={d}, workers={d}, retained={Bi:.2}", .{ @tagName(platform.target), request_size, + opts.direct_io, + reads.alignment, block_size, self.dma_budget_bytes, width, @@ -251,7 +266,7 @@ pub const Loader = struct { try self.createDestinations(batch, specs, progress); break :batch batch; }; - Planner.publishFiles(&self.scheduler, self.io, batch, batch.parts, self.plan_config) catch |err| { + Planner.publishFiles(&self.scheduler, self.io, batch, batch.parts, self.plan_config, self.reads) catch |err| { // Fail the loader, seal and await what was published of this // batch, and return the loader's first error. self.recordError(err, "plan batch {d}, published_plans={d}", .{ batch.sequence, batch.plans.items.len }); @@ -685,10 +700,14 @@ const Part = struct { }; /// One source file: every job of a plan reads the file of its slot, which -/// the first read opens. +/// the first read opens, or the planner when it asks the VFS whether the +/// file is read directly (`Planner.Reads`). const SourceSlot = struct { uri: []const u8, file: LazyOnce(std.Io.File) = .{}, + /// The file's read alignment once the planner decided it + /// (`Planner.Reads`): 0 when it is read exactly. + alignment: ?usize = null, fn ensure(self: *SourceSlot, io: std.Io) std.Io.File.OpenError!std.Io.File { const file = try self.file.ensure(io, std.Io.Dir.openFile, .{ .cwd(), io, self.uri, .{ .mode = .read_only } }); @@ -723,8 +742,8 @@ const Plan = struct { }; /// `len` bytes from the job's `file_offset`, of which the first - /// `minimum_len` hold tensor data and must exist. Reads are exact: - /// the two are equal. + /// `minimum_len` hold tensor data and must exist. The rest is + /// alignment padding the end of the file may cut. const Read = struct { len: usize, minimum_len: usize, @@ -790,10 +809,53 @@ const Planner = struct { request_size: usize, }; + /// How each file is read: aligned when it is local and the VFS that + /// opens it reads it directly under `policy`, decided once per file and + /// kept in its slot. A remote file is read exact without being opened + /// here. Without a VFS (`io` is none, or no alignment) every file takes + /// `alignment`. + /// Widening buffered reads is not free: a widened read served from the + /// page cache costs the DMA sources their block alignment for nothing, + /// and warm replicated Llama on four GB300 lost 3-6% when its first + /// tensor no longer landed at block offset zero. So a file asks the VFS + /// actually owning its handle (`VFS.fromIo`, in `init`), both to avoid + /// that cost and to avoid indexing another VFS's handle table; an `io` + /// that is no VFS never reads directly. + const Reads = struct { + io: std.Io, + vfs: ?*VFS, + policy: VFS.DirectIo, + /// Every read of a file read directly starts and ends at a multiple + /// of this; 0 reads every file exact. + alignment: usize, + + fn init(io: std.Io, policy: VFS.DirectIo) Reads { + const alignment = backend.readAlignment(io, policy); + return .{ .io = io, .vfs = if (alignment != 0) VFS.fromIo(io) else null, .policy = policy, .alignment = alignment }; + } + + /// The file's alignment, decided the first time it is asked: a + /// local file is opened here. One that fails to open is read exact: + /// its reads fail with the same error (`LazyOnce`), which `readSlot` + /// logs with the path. + fn alignmentOf(self: Reads, slot: *SourceSlot) error{Canceled}!usize { + const vfs = self.vfs orelse return self.alignment; + if (slot.alignment) |known| return known; + const direct = vfs.isLocal(slot.uri) and vfs.useDirectIo(slot.ensure(self.io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return 0, + }, self.policy); + const alignment = if (direct) self.alignment else 0; + slot.alignment = alignment; + return alignment; + } + }; + /// Plans `parts` one file at a time and publishes each plan as soon as /// it exists, so workers claim the first file while the rest is planned, /// each plan in the batch's arena. Stops at the first error; the caller - /// seals or fails the batch. + /// seals or fails the batch. `reads` says how each file is read + /// (`Reads`). /// Whole-model planning delayed the first read by 0.32 s for DeepSeek's /// 46 files on one MI300X; publishing per file overlaps the rest with IO. fn publishFiles( @@ -802,6 +864,7 @@ const Planner = struct { batch: *Batch, parts: []Part, config: Config, + reads: Reads, ) !void { const order = try sortedPartOrder(scheduler.allocator, parts); defer scheduler.allocator.free(order); @@ -809,7 +872,7 @@ const Planner = struct { while (file_start < order.len) { const file_end = fileGroupEnd(parts, order, file_start); const planning_started: std.Io.Timestamp = .now(io, .awake); - const plan = try preparePlan(scheduler.allocator, batch.arena.allocator(), parts, order[file_start..file_end], config); + const plan = try preparePlan(scheduler.allocator, batch.arena.allocator(), parts, order[file_start..file_end], config, reads); plan.planning_ns = @intCast(@max(planning_started.untilNow(io, .awake).nanoseconds, 0)); try scheduler.publish(io, batch, plan); if (batch.published_at == null) batch.published_at = .now(io, .awake); @@ -818,22 +881,27 @@ const Planner = struct { } /// Plans one file: `order` indexes its `parts` by offset and size - /// (`sortedPartOrder`), and `JobCut.cutRuns` cuts them. The plan goes in - /// `arena` (`JobCut.finish`), the cut in `allocator`. + /// (`sortedPartOrder`), and `JobCut.cutRuns` cuts them at the file's + /// alignment (`Reads.alignmentOf`). The plan goes in `arena` + /// (`JobCut.finish`), the cut in `allocator`. fn preparePlan( allocator: std.mem.Allocator, arena: std.mem.Allocator, parts: []Part, order: []const usize, config: Config, + reads: Reads, ) !*Plan { const source_slot = parts[order[0]].source_slot; for (order) |part_index| std.debug.assert(std.mem.eql(u8, parts[part_index].source.file_uri, parts[order[0]].source.file_uri)); + const alignment = try reads.alignmentOf(source_slot); var cut: JobCut = .{ .allocator = allocator, .parts = parts, .source_slot = source_slot, .config = config, + .alignment = alignment, + .maximum_job_len = maximumJobLen(config.request_size, alignment), }; defer cut.deinit(); // Jobs are claimed in planning order, which is file order. A @@ -855,6 +923,9 @@ const Planner = struct { /// The file every job of the plan reads. source_slot: *SourceSlot, config: Config, + /// The alignment of the plan's file (`Reads.alignmentOf`). + alignment: usize, + maximum_job_len: usize, jobs: std.ArrayList(Plan.Job) = .empty, transfers: std.ArrayList(Plan.Transfer) = .empty, safe_boundaries: std.ArrayList(u64) = .empty, @@ -890,13 +961,17 @@ const Planner = struct { /// Cuts the runs of `order` (part indices in file order) into jobs: /// each run of touching or overlapping ranges into the minimum number - /// of jobs of at most a request, cut at part-safe boundaries where - /// that number allows. No job depends on another, since every DMA - /// piece is submitted as soon as its block is read. + /// of jobs, cut at part-safe boundaries where that number allows. No + /// job depends on another, since every DMA piece is submitted as soon + /// as its block is read. With an alignment (`Reads.alignmentOf`), a + /// job's read widens to aligned bounds around its range + /// (`maximumJobLen` leaves room for it), and the transfers address + /// the widened read. fn cutRuns(self: *JobCut, order: []const usize) !void { const allocator = self.allocator; const parts = self.parts; - const maximum_job_len = self.config.request_size; + const alignment = self.alignment; + const maximum_job_len = self.maximum_job_len; var run_cursor: usize = 0; while (run_cursor < order.len) { self.safe_boundaries.clearRetainingCapacity(); @@ -960,6 +1035,9 @@ const Planner = struct { break :boundary selected orelse maximum_end; }; const job_len: usize = @intCast(job_end - job_start); + const read_start = if (alignment == 0) job_start else std.mem.alignBackward(u64, job_start, alignment); + const read_end = if (alignment == 0) job_end else std.mem.alignForward(u64, job_end, alignment); + const read_len: usize = @intCast(read_end - read_start); const transfer_start = self.transfers.items.len; while (candidate_start < run_part_end) { const candidate = parts[order[candidate_start]].fileRange(); @@ -979,14 +1057,14 @@ const Planner = struct { .start = logical_start, .end = logical_start + (intersection_end - intersection_start), .file_offset = intersection_start, - }, job_start); + }, read_start); } std.debug.assert(self.transfers.items.len != transfer_start); try self.jobs.append(allocator, .{ - .file_offset = job_start, + .file_offset = read_start, .transfer_start = transfer_start, .transfer_len = self.transfers.items.len - transfer_start, - .kind = .{ .dma_read = .{ .len = job_len, .minimum_len = job_len } }, + .kind = .{ .dma_read = .{ .len = read_len, .minimum_len = @intCast(job_end - read_start) } }, }); self.source_bytes +|= @intCast(job_len); job_start = job_end; @@ -1065,6 +1143,17 @@ const Planner = struct { return len / block_size + @intFromBool(len % block_size != 0); } + /// The longest tensor range one job may cover: the request size, less + /// the two alignment units its widened read can add, so a widened job + /// still fits a request (a credit's slot). `backend.requestSize` refuses + /// a request that cannot hold them. + fn maximumJobLen(request_size: usize, alignment: usize) usize { + std.debug.assert(request_size != 0); + if (alignment == 0) return request_size; + std.debug.assert(std.math.isPowerOfTwo(alignment) and 2 * alignment < request_size); + return request_size - 2 * alignment; + } + /// Part indices sorted by file URI, offset, size and index: the /// planner's input, taken one file group at a time (`fileGroupEnd`), in /// file order. @@ -1992,10 +2081,15 @@ fn writeTestFile(dir: std.Io.Dir, io: std.Io, name: []const u8, contents: []cons // Planner tests. -/// `Planner.preparePlan`, kept in `arena`; the testing allocator checks the -/// cut for leaks. -fn prepareTestPlan(arena: std.mem.Allocator, parts: []Part, order: []const usize, config: Planner.Config) !*Plan { - return Planner.preparePlan(std.testing.allocator, arena, parts, order, config); +/// Every file read at `alignment`, none opened to ask. +fn fixedReads(alignment: usize) Planner.Reads { + return .{ .io = undefined, .vfs = null, .policy = .off, .alignment = alignment }; +} + +/// `Planner.preparePlan` with every file read at `alignment`, kept in +/// `arena`; the testing allocator checks the cut for leaks. +fn prepareTestPlan(arena: std.mem.Allocator, parts: []Part, order: []const usize, config: Planner.Config, alignment: usize) !*Plan { + return Planner.preparePlan(std.testing.allocator, arena, parts, order, config, fixedReads(alignment)); } /// One plain destination per source, as `createDestinations` builds them @@ -2039,6 +2133,36 @@ fn discardTestBatch(scheduler: *Scheduler, batch: *Batch) void { batch.destroy(); } +test "the planner opens only local files to ask about direct I/O" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents: [2 * 4096]u8 = @splat(7); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "drafter.bin", &contents, &path_buffer); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + // A scheme whose files the plain io cannot open: asking would fail. + try vfs.registerBackend("remote", .{ .io = io }); + const vfs_io = vfs.io(); + const reads: Planner.Reads = .{ .io = vfs_io, .vfs = &vfs, .policy = .on, .alignment = 4096 }; + + // A remote file is read exact, without an open on the planner. + var remote: SourceSlot = .{ .uri = "remote://bucket/model.safetensors" }; + defer remote.deinit(vfs_io); + try std.testing.expectEqual(@as(usize, 0), try reads.alignmentOf(&remote)); + try std.testing.expect(remote.file.readyValue() == null); + + // A local file beside it is opened and asked, whatever the checkpoint: + // direct where the filesystem allows it. + var local: SourceSlot = .{ .uri = path }; + defer local.deinit(vfs_io); + const alignment = try reads.alignmentOf(&local); + const file = local.file.readyValue().?.*; + try std.testing.expectEqual(@as(usize, if (vfs.useDirectIo(file, .on)) 4096 else 0), alignment); +} + test "source planner coalesces exact adjacent and overlapping tensor ranges per file" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -2068,7 +2192,7 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per try Planner.publishFiles(&scheduler, io, batch, &parts, .{ .block_size = 4, .request_size = 8, - }); + }, fixedReads(0)); // One plan per file. a:[10,18) merges adjacency and the duplicate, // a:[20,24) remains exact, and b:[3,15) is split at the request-size @@ -2119,12 +2243,35 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per const aligned_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ .block_size = 4, .request_size = 8, - }); + }, 0); try std.testing.expectEqual(@as(usize, 3), aligned_plan.jobs.len); try std.testing.expectEqual(@as(usize, 6), aligned_plan.transfers.len); try std.testing.expectEqual(@as(usize, 7), aligned_plan.jobs[0].kind.dma_read.len); try std.testing.expectEqual(@as(usize, 8), aligned_plan.jobs[1].kind.dma_read.len); try std.testing.expectEqual(@as(usize, 5), aligned_plan.jobs[2].kind.dma_read.len); + + // Widened for direct I/O: the same three jobs (16 - 2 * 4 = 8 still cuts + // at the tensor-safe boundaries 7 and 15), each read at 4-byte bounds, + // its transfers relative to the widened start; the last read runs past + // the tensor data, of which every byte is required. + const widened_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ + .block_size = 4, + .request_size = 16, + }, 4); + try std.testing.expectEqual(@as(usize, 3), widened_plan.jobs.len); + try std.testing.expectEqual(@as(u64, 0), widened_plan.jobs[0].file_offset); + try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 7), widened_plan.jobs[0].kind.dma_read.minimum_len); + try std.testing.expectEqual(@as(u64, 4), widened_plan.jobs[1].file_offset); + try std.testing.expectEqual(@as(usize, 12), widened_plan.jobs[1].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 11), widened_plan.jobs[1].kind.dma_read.minimum_len); + try std.testing.expectEqual(@as(usize, 3), widened_plan.jobs[1].blockLen(4)); + try std.testing.expectEqual(@as(usize, 3), widened_plan.transfers[widened_plan.jobs[1].transfer_start].landing); + try std.testing.expectEqual(@as(u64, 12), widened_plan.jobs[2].file_offset); + try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[2].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[2].kind.dma_read.minimum_len); + try std.testing.expectEqual(@as(usize, 3), widened_plan.transfers[widened_plan.jobs[2].transfer_start].landing); + try std.testing.expectEqual(@as(u64, 20), widened_plan.source_bytes); } // Scheduler tests. @@ -2206,11 +2353,12 @@ test "scheduler publishes a submission one file at a time and claims the files i try Planner.publishFiles(&scheduler, io, batch, &parts, .{ .block_size = 4, .request_size = 4, - }); + }, fixedReads(0)); try std.testing.expectEqual(@as(usize, 2), batch.plans.items.len); try std.testing.expectEqual(@as(usize, 2), batch.plans.items[0].jobs.len); try std.testing.expectEqual(@as(usize, 1), batch.plans.items[1].jobs.len); try std.testing.expectEqual(@as(usize, 3), unclaimedJobs(&scheduler)); + try std.testing.expect(batch.published_at != null); // File a's jobs in offset order, then file b's. var claim = scheduler.claim(io).?; @@ -2445,6 +2593,7 @@ const TestLoader = struct { .dma_budget_bytes = 64, .created_at = .now(std.testing.io, .awake), .plan_config = .{ .block_size = 64, .request_size = 64 }, + .reads = fixedReads(0), }; } diff --git a/zml/io/dma_calibration.zig b/zml/io/dma_calibration.zig index 962224e1..dd4e0e30 100644 --- a/zml/io/dma_calibration.zig +++ b/zml/io/dma_calibration.zig @@ -1,13 +1,13 @@ //! DMA sizing for the direct backend: the block size, decided once when the //! backend is created. It screens `default_block_sizes` (2 to 32 MiB), or -//! the one block the backend's options pin (`backend.Options.dmaBlockSizes`), -//! on one representative device, at `depth` transfers per device and over -//! fixed windows, and keeps the smallest block within 8% of the measured -//! peak; borderline candidates get longer, alternating confirmation -//! windows. The ring it measures in stays mapped as the load's first pinned -//! capacity. CPU returns the smallest candidate without measuring: its -//! plugin's transfer is a memcpy, and a load takes the same time at every -//! block size. The buffered backend has nothing to size. +//! the one block `zml.io.next.Loader.Options.dma_block_size` gives, on one +//! representative device, at `depth` transfers per device and over fixed +//! windows, and keeps the smallest block within 8% of the measured peak; +//! borderline candidates get longer, alternating confirmation windows. The +//! ring it measures in stays mapped as the load's first pinned capacity. +//! CPU returns the smallest candidate without measuring: its plugin's +//! transfer is a memcpy, and a load takes the same time at every block +//! size. The buffered backend has nothing to size. const std = @import("std"); const pjrt = @import("pjrt"); diff --git a/zml/io/loader.zig b/zml/io/loader.zig index 26833c51..8a0f19db 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -27,6 +27,8 @@ //! pending: one program group is in flight at a time. const std = @import("std"); +const VFS = @import("vfs"); + const Buffer = @import("../buffer.zig").Buffer; const Compiler = @import("../Compiler.zig"); const Exe = @import("../exe.zig").Exe; @@ -111,14 +113,41 @@ pub const Loader = struct { stats: Stats = .{}, pub const Options = struct { + /// The checkpoint's path or URI, read by `init` only. When `io` is a + /// VFS, the VFS backend serving it sizes the reads + /// (`VFS.readHints`). Null, or a plain `io`: local sizing. + checkpoint: ?[]const u8 = null, /// Concurrent source reads, 1 to `max_read_parallelism`, fixed for - /// the load. Local checkpoints read fastest at 12 to 16 on every - /// host measured. - read_parallelism: usize = 16, + /// the load. Null: 16 for a local checkpoint, 32 for a high-latency + /// one. The direct backend may lower it to fit its pinned memory + /// (logged at debug). + read_parallelism: ?usize = null, + /// The direct backend's DMA block, at most 32 MiB, and a multiple of + /// `VFS.direct_io_alignment` when `io` is a VFS and `direct_io` is + /// not `off`. Null: calibration picks one of + /// `dma_calibration.default_block_sizes`, which suits every measured + /// host. A tuning knob, ignored by the buffered backend; `init` + /// refuses a block the checkpoint's reads cannot use. + dma_block_size: ?usize = null, + /// Direct I/O for local source files, decided once per file per + /// loader (`VFS.DirectIo` for the page-cache trade-off). Nothing + /// changes when `io` is not a VFS. + direct_io: VFS.DirectIo = .auto, /// Output bytes of the program tensors one group holds at most, one /// tensor at least. A group's inputs are read together and freed /// once its programs ran. program_group_bytes: usize = 8 << 30, + + /// The default read widths by latency class, from the recorded width + /// sweeps. Local sources knee at 12 to 16 on every host measured: the + /// B70 peaks at 12 to 16 and loses 11% at 24, one MI300X prefers 12, + /// the RTX 5090 host 16 over 12, and the GB300 plateau from 12 to 48 + /// has 16 within 5% of its best rung, so 16 is the width with the + /// smallest regret. Remote sources plateau from 24 to 32: hf:// reads + /// 32 and 64 alike and AWS was flat from 24, while pinned memory grows + /// with the width (32 x 32 MiB requests pin 1.5 GiB). + const local_read_parallelism: usize = 16; + const high_latency_read_parallelism: usize = 32; }; /// Concurrent source reads at most (`Options.read_parallelism`). @@ -128,15 +157,19 @@ pub const Loader = struct { /// creating the backend (for the direct one: DMA calibration and the /// pinned pre-growth) run on a task of the loader's own, or here when /// `io` has no concurrency; the first `start` waits for that task and - /// returns its error. A read width out of range is `InvalidOptions`, - /// logged. + /// returns its error. A read width or a block size out of range, a + /// block that is not a multiple of the direct-I/O alignment + /// (`Options.dma_block_size`), a checkpoint whose scheme the VFS does + /// not serve, a block calibration may select whose request (the read + /// chunk rounded up to whole blocks) passes 32 MiB, or a request too + /// short for a widened direct read is `InvalidOptions`, logged. pub fn init( allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options, ) error{InvalidOptions}!Loader { - const backend_opts = try backendOptions(opts); + const backend_opts = try backendOptions(io, opts); const setup_args = .{ allocator, io, platform, backend_opts }; return .{ .allocator = allocator, @@ -801,14 +834,47 @@ fn checkPlacement(platform: *const Platform, target: Target, from: []const u8) e } } -/// `opts` as the backends take them, with the local read hints. Every -/// option is checked here, once. -fn backendOptions(opts: Loader.Options) error{InvalidOptions}!backend.Options { - if (opts.read_parallelism == 0 or opts.read_parallelism > Loader.max_read_parallelism) { - load_log.err("invalid loader options: read_parallelism={d}, expected 1..{d}: InvalidOptions", .{ opts.read_parallelism, Loader.max_read_parallelism }); +/// `opts` as the backends take them, the checkpoint resolved to the read +/// hints of the VFS backend serving it and the read width to its default. +/// Every option is checked here, once. +fn backendOptions(io: std.Io, opts: Loader.Options) error{InvalidOptions}!backend.Options { + const hints: VFS.ReadHints = hints: { + const path = opts.checkpoint orelse break :hints .local; + const vfs = VFS.fromIo(io) orelse break :hints .local; + break :hints vfs.readHints(path) catch |err| { + load_log.err("invalid loader options: checkpoint={s}: {s}", .{ path, @errorName(err) }); + return error.InvalidOptions; + }; + }; + if (opts.read_parallelism) |width| { + if (width == 0 or width > Loader.max_read_parallelism) { + load_log.err("invalid loader options: read_parallelism={d}, expected 1..{d}: InvalidOptions", .{ width, Loader.max_read_parallelism }); + return error.InvalidOptions; + } + } + const max_request = backend.max_read_request_size; + if (hints.read_chunk_size == 0 or hints.read_chunk_size > max_request) { + load_log.err("invalid loader options: checkpoint={?s}, read_chunk_size={d}, expected 1..{d}: InvalidOptions", .{ opts.checkpoint, hints.read_chunk_size, max_request }); return error.InvalidOptions; } - return .{ .read_parallelism = opts.read_parallelism }; + const options: backend.Options = .{ + .read_parallelism = opts.read_parallelism orelse if (hints.high_latency) + Loader.Options.high_latency_read_parallelism + else + Loader.Options.local_read_parallelism, + .read_hints = hints, + .dma_block_size = opts.dma_block_size, + .direct_io = opts.direct_io, + }; + // Checked for every block calibration may select. + const alignment = backend.readAlignment(io, opts.direct_io); + for (options.dmaBlockSizes()) |block_size| { + if (backend.requestSize(hints.read_chunk_size, block_size, alignment) == null) { + load_log.err("invalid loader options: checkpoint={?s}, read_chunk_size={d}, dma_block_size={d}, alignment={d}, max_request={d}: no request size fits: InvalidOptions", .{ opts.checkpoint, hints.read_chunk_size, block_size, alignment, max_request }); + return error.InvalidOptions; + } + } + return options; } /// A program must compute one tensor of `expected`, placed on the same @@ -1213,16 +1279,44 @@ test "init checks every option and returns InvalidOptions" { const allocator = std.testing.allocator; const io = std.testing.io; const platform = testing.env(); - for ([_]usize{ 0, Loader.max_read_parallelism + 1 }) |width| { - if (Loader.init(allocator, io, platform, .{ .read_parallelism = width })) |value| { + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + const Case = struct { io: std.Io, opts: Loader.Options }; + const cases = [_]Case{ + .{ .io = io, .opts = .{ .read_parallelism = 0 } }, + .{ .io = io, .opts = .{ .read_parallelism = Loader.max_read_parallelism + 1 } }, + .{ .io = io, .opts = .{ .dma_block_size = 0 } }, + .{ .io = io, .opts = .{ .dma_block_size = backend.max_read_request_size + 1 } }, + // No backend of the VFS serves the scheme. + .{ .io = vfs.io(), .opts = .{ .checkpoint = "missing://bucket/model" } }, + }; + for (cases) |case| { + if (Loader.init(allocator, case.io, platform, case.opts)) |value| { var loader = value; loader.deinit(); return error.TestUnexpectedResult; } else |err| try std.testing.expectEqual(error.InvalidOptions, err); } - // Nothing started, so `deinit` cancels the set-up or destroys what it - // built. - var loader = try Loader.init(allocator, io, platform, .{}); + // Under a VFS, a local file may be read directly: a block is a whole + // number of alignment units, reads landing in blocks. + if (VFS.direct_io_alignment != 0) { + if (Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = "/model.safetensors", .dma_block_size = (8 << 20) + VFS.direct_io_alignment / 2 })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidOptions, err); + // Whatever the checkpoint: a request of one alignment unit cannot + // hold a direct read widened at each end. + try vfs.registerBackend("tiny", .{ .io = io, .read_hints = .{ .read_chunk_size = VFS.direct_io_alignment } }); + if (Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = "tiny://model", .dma_block_size = VFS.direct_io_alignment })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidOptions, err); + } + // A plain io has no VFS to ask: local sizing. Nothing started, so + // `deinit` cancels the set-up or destroys what it built. + var loader = try Loader.init(allocator, io, platform, .{ .checkpoint = "missing://bucket/model" }); loader.deinit(); } @@ -1442,7 +1536,7 @@ test "a program takes its inputs from sources, in order" { /// through the direct backend, and this keeps the front end over the /// buffered one under test. fn initBuffered(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Loader.Options) !Loader { - const backend_opts = try backendOptions(opts); + const backend_opts = try backendOptions(io, opts); return .{ .allocator = allocator, .io = io, From 388e5ed9df47720255668698e1cb53393f7032e8 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 15:14:52 +0100 Subject: [PATCH 082/103] zml/io: load composite destinations and zero-fill gaps A source lowered into several parts, some of them strided, now loads as one destination. The front end expands a strided part of at most 64 runs, or of runs of 1 MiB or more, into one contiguous part per run, sorts the parts by destination offset and checks them; a spec with any other strided part is still refused. Padding is no longer refused on the platforms that load through the direct backend. The direct backend fills every device's placement exactly: the bytes no part covers go out with the plan of the destination's first part, as a last fill job that reads nothing and transfers from the zero block, cut by blocks and merged per writer set like read bytes. The batch planning debug line counts fill jobs apart from source jobs. The new tests replay planned composites over four and eight CPU devices, sharded by rows, by columns or replicated, and check that every device byte is written once, from its source or from zeros; the front end's tests load concatenated, padded, interleaved, stacked and split sources, and the buffered backend loads a padded one. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 409 ++++++++++++++++++++++++++++++++++++--- zml/io/loader.zig | 389 +++++++++++++++++++++++++++++++++---- 2 files changed, 730 insertions(+), 68 deletions(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index dff6ebd7..6f4660b4 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -45,7 +45,9 @@ //! offsets, which the destination's `ShardGrid` places on the devices by //! arithmetic. Pieces of one destination merge across parts when block, //! writers and both offsets are contiguous, and its parts may come from -//! different files and plans. +//! different files and plans. Bytes no part covers go out with the plan of +//! its first part, as a last fill job that reads nothing and transfers from +//! the zero block, which no credit owns. //! //! Locks. Where several pumps are held, they are taken in device order. A //! pump's mutex may be held while a released request gives its credit back @@ -64,6 +66,7 @@ const pjrtx = @import("../pjrtx.zig"); const Platform = @import("../platform.zig").Platform; const safetensors = @import("../safetensors.zig"); const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); const backend = @import("backend.zig"); const BackendOptions = backend.Options; const LoadPart = backend.LoadPart; @@ -94,8 +97,8 @@ pub const Loader = struct { /// The pinned arenas, grown once at creation and cut into `blocks`. workspace: host_memory.Workspace, /// The workspace's blocks (`carve`): the zero block first, then the - /// credits' slots (`Credits`). The zero block is in no credit's slot - /// and is never written after `create`. + /// credits' slots (`Credits`). The zero block is the source of every DMA + /// fill transfer, in no credit's slot, never written again. blocks: []const host_memory.Block, scheduler: Scheduler, metrics: Metrics = .{}, @@ -167,7 +170,7 @@ pub const Loader = struct { dma_calibration.depth * platform.devices.len, Planner.blockCount(backend.max_read_request_size, block_size), ); - // One more block for the zeros, which no credit owns. + // One more block for the zero fill, which no credit owns. const zero_blocks = 1; const available = workspace.capacity(block_size); const width = @min( @@ -253,11 +256,10 @@ pub const Loader = struct { /// batch: work on the first file starts while the rest is planned. The /// batch owns its destinations and parts until `awaitBatch` retires it. /// The specs keep the `LoadSpec` contract, which the front end checked, - /// and this backend's: device memory, and contiguous parts covering - /// every byte of their destination. A planning failure fails the - /// loader (a partial submission can never complete): the batch is - /// awaited here and the caller sees only the first error. A failed - /// loader refuses every submit with its first error. + /// and this backend's: device memory and contiguous parts. A planning + /// failure fails the loader (a partial submission can never complete): + /// the batch is awaited here and the caller sees only the first error. + /// A failed loader refuses every submit with its first error. pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { try self.checkOpen(); const batch = batch: { @@ -363,7 +365,7 @@ pub const Loader = struct { /// Creates the batch's destinations, one per spec, and their parts in /// its arena, their sources and dimensions staying the specs' until the /// batch is awaited (`backend.LoadSpec`). A destination's device state - /// is created lazily by the first worker that reads for it. + /// is created lazily by the first worker that reads or fills for it. fn createDestinations( self: *Loader, batch: *Batch, @@ -392,9 +394,6 @@ pub const Loader = struct { }; covered_bytes += part.lattice.width; } - // The parts are disjoint (`backend.LoadSpec`): they cover the - // destination when their widths add up to it. - std.debug.assert(covered_bytes == grid.total); // submit destination.* = .{ .shape = spec.shape, .grid = grid, @@ -513,25 +512,30 @@ pub const Loader = struct { var jobs: usize = 0; var transfers: usize = 0; var dma_submissions: usize = 0; + var kinds: std.EnumArray(std.meta.Tag(Plan.Job.Kind), usize) = .initFill(0); for (batch.plans.items) |plan| { planning_ns += plan.planning_ns; source_bytes += plan.source_bytes; jobs += plan.jobs.len; transfers += plan.transfers.len; dma_submissions += plan.events.len; + for (plan.jobs) |job| kinds.getPtr(job.kind).* += 1; } - const average_read_size = if (jobs == 0) 0 else source_bytes / jobs; - const coalescing_ratio = if (jobs == 0) + const fill_jobs = kinds.get(.dma_fill); + const source_jobs = jobs - fill_jobs; + const average_read_size = if (source_jobs == 0) 0 else source_bytes / source_jobs; + const coalescing_ratio = if (source_jobs == 0) 0 else @as(f64, @floatFromInt(batch.parts.len)) / - @as(f64, @floatFromInt(jobs)); - load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ + @as(f64, @floatFromInt(source_jobs)); + load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ batch.sequence, batch.plans.items.len, @as(f64, @floatFromInt(planning_ns)) / std.time.ns_per_s, source_bytes, - jobs, + source_jobs, + fill_jobs, batch.parts.len, transfers, dma_submissions, @@ -641,10 +645,10 @@ pub const Batch = struct { }; /// One output buffer and what places it. Owned by the batch. Its parts may -/// come from several files, so several plans can write it: their pieces all -/// count toward the same targets (`Target.total` is the placement's bytes, -/// whichever plan submits them), and its device state is created once, by -/// the first worker that reads for it. +/// come from several files, so several plans can write it: their pieces, +/// and the zero fill, all count toward the same targets (`Target.total` is +/// the placement's bytes, whichever plan submits them), and its device +/// state is created once, by the first worker that reads or fills for it. const Destination = struct { /// The caller's shape, which the output buffer keeps. shape: Shape, @@ -653,7 +657,9 @@ const Destination = struct { grid: ShardGrid, output: *Buffer, /// What fills it, in ascending, disjoint destination order - /// (`backend.LoadSpec`). + /// (`backend.LoadSpec`): a slice of the batch's parts. Bytes no part + /// covers are written from the loader's zero block, by the fill job of + /// the plan holding `parts[0]`. parts: []Part, /// Bytes not read yet, the parts' widths at first. The read that /// brings it to zero completes the destination's item on `progress`. @@ -728,7 +734,8 @@ const Plan = struct { /// A claimable job: its slice of the plan's transfers and what its /// worker does for them. Its request slot is its index. const Job = struct { - /// Where the job's read starts. + /// Where the job's read starts; zero for a fill job, which reads + /// nothing and comes after the plan's reads. file_offset: u64, transfer_start: usize, transfer_len: usize, @@ -739,6 +746,9 @@ const Plan = struct { /// Read into pinned blocks in file order, where its transfers /// address it, then copied to the devices by DMA. dma_read: Read, + /// Reads nothing: every transfer goes from the loader's zero + /// block (`Planner.JobCut.appendFillTransfers`). + dma_fill, }; /// `len` bytes from the job's `file_offset`, of which the first @@ -749,17 +759,20 @@ const Plan = struct { minimum_len: usize, }; - /// The blocks the job reads into. + /// The blocks of its credit's slot the job reads into; none for a + /// fill job. fn blockLen(self: Job, block_size: usize) usize { return switch (self.kind) { .dma_read => |read| Planner.blockCount(read.len, block_size), + .dma_fill => 0, }; } }; /// `len` bytes at `landing` of the job's blocks taken end to end, for /// every device of `writer_mask` at `destination_offset` of its shard. - /// Every transfer stays inside one block. + /// Every transfer stays inside one block. A fill one lands at 0 and + /// reads the zero block. const Transfer = struct { destination: *Destination, landing: usize, @@ -881,9 +894,10 @@ const Planner = struct { } /// Plans one file: `order` indexes its `parts` by offset and size - /// (`sortedPartOrder`), and `JobCut.cutRuns` cuts them at the file's - /// alignment (`Reads.alignmentOf`). The plan goes in `arena` - /// (`JobCut.finish`), the cut in `allocator`. + /// (`sortedPartOrder`), `JobCut.cutRuns` cuts them at the file's + /// alignment (`Reads.alignmentOf`), then `JobCut.appendFillJob` appends + /// the zero fill. The plan goes in `arena` (`JobCut.finish`), the cut in + /// `allocator`. fn preparePlan( allocator: std.mem.Allocator, arena: std.mem.Allocator, @@ -909,6 +923,10 @@ const Planner = struct { // it on the two hosts where the DMA pump is not the ceiling and made // no difference. try cut.cutRuns(order); + // The zero fill of each destination goes out once, with the plan + // of its first part; it depends on no read, so one job after this + // file's reads carries all of it. + try cut.appendFillJob(order); return cut.finish(arena); } @@ -1138,6 +1156,80 @@ const Planner = struct { for (transfers) |transfer| count += @popCount(transfer.writer_mask); return count; } + + /// The zero fill of the destinations whose first part is among + /// `order`, the file's parts, as one job that reads one block, the + /// zero block. + fn appendFillJob(self: *JobCut, order: []const usize) !void { + const fill_start = self.transfers.items.len; + for (order) |part_index| { + const part = &self.parts[part_index]; + const destination = part.destination; + if (part != &destination.parts[0]) continue; + try self.appendFillTransfers(fill_start, destination); + } + const fill_transfers = self.transfers.items[fill_start..]; + if (fill_transfers.len == 0) return; + try self.jobs.append(self.allocator, .{ + .file_offset = 0, + .transfer_start = fill_start, + .transfer_len = fill_transfers.len, + .kind = .dma_fill, + }); + } + + /// Appends the zero-fill transfers of `destination`: every logical + /// range no part covers (the gaps between its parts and the tail), + /// placed by its grid like read bytes, each piece at most one block + /// at landing 0, where the job reads the zero block. Pieces count + /// toward their targets' totals, so the fill completes each + /// device's placement bytes alongside the parts. + fn appendFillTransfers(self: *JobCut, transfer_start: usize, destination: *Destination) !void { + var covered_end: u64 = 0; + for (destination.parts) |part| { + const lattice = part.lattice; + if (lattice.dst_offset > covered_end) try self.appendFillRange(transfer_start, destination, covered_end, lattice.dst_offset); + covered_end = lattice.dst_offset + lattice.width; + } + if (covered_end < destination.grid.total) try self.appendFillRange(transfer_start, destination, covered_end, destination.grid.total); + } + + /// The fill of the logical range `[start, end)`: one range of each box + /// it touches, cut by blocks, merged where one writer set's pieces + /// are contiguous. + fn appendFillRange(self: *JobCut, transfer_start: usize, destination: *Destination, start: u64, end: u64) !void { + const block_size = self.config.block_size; + const grid = &destination.grid; + for (0..grid.box_count) |box| { + var destination_offset: usize = @intCast(grid.rankIn(box, start)); + var remaining: usize = @intCast(grid.rankIn(box, end) - destination_offset); + const writer_mask = grid.writers[box]; + while (remaining != 0) { + if (self.transfers.items.len > transfer_start) merge: { + const previous = &self.transfers.items[self.transfers.items.len - 1]; + if (previous.destination != destination or previous.writer_mask != writer_mask or + previous.destination_offset + previous.len != destination_offset or + previous.len == block_size) + break :merge; + const grown = @min(remaining, block_size - previous.len); + previous.len += grown; + destination_offset += grown; + remaining -= grown; + continue; + } + const take = @min(remaining, block_size); + try self.transfers.append(self.allocator, .{ + .destination = destination, + .landing = 0, + .writer_mask = writer_mask, + .destination_offset = destination_offset, + .len = take, + }); + destination_offset += take; + remaining -= take; + } + } + } }; fn blockCount(len: usize, block_size: usize) usize { return len / block_size + @intFromBool(len % block_size != 0); @@ -1369,6 +1461,8 @@ const ReadRequest = struct { const blocks = loader.credits.slotBlocks(self.slot)[0..job.blockLen(block_size)]; switch (job.kind) { .dma_read => |read| try self.readBlocks(scratch, blocks, read), + // Reads nothing: its transfers read the zero block. + .dma_fill => return Dma.enqueue(self, loader.blocks[0..1]), } for (transfers) |transfer| transfer.destination.recordRead(transfer.len); try Dma.enqueue(self, blocks); @@ -1828,7 +1922,8 @@ const DestinationState = struct { manager: *pjrt.AsyncHostToDeviceTransferManager, device_index: usize, /// Bytes of the placement on this device. The pieces partition them: - /// those of every part, from whichever file and plan. + /// those of every part, from whichever file and plan, and of the + /// zero fill. total: usize, /// Owned by the pump; read by the awaiting task once the batch is /// done and nothing submits any more. @@ -1858,7 +1953,7 @@ const DestinationState = struct { /// Creates the destination's device buffers and transfer managers, one /// per device of its grid in canonical order, and writes the output /// shell. Runs once per destination (`LazyOnce`), however many parts - /// feed it, on the worker that first reads for it. + /// feed it, on the worker that first reads or fills for it. fn init(allocator: std.mem.Allocator, platform: *const Platform, destination: *const Destination) StateInitError!DestinationState { const grid = &destination.grid; const targets = try allocator.alloc(Target, grid.devices.len); @@ -2274,6 +2369,258 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per try std.testing.expectEqual(@as(u64, 20), widened_plan.source_bytes); } +/// Plans a composite destination fed by three parts from two files, with a +/// gap and a tail of zero fill, beside a plain one, then replays the plans on +/// host memory: every device's placement bytes are written exactly once, +/// from the right source bytes or from zeros. +const CompositeTest = struct { + const Options = struct { + block_size: usize, + request_size: usize, + /// Whether some piece holds a's last byte and b's first one. + cross_part_piece: bool, + }; + + /// The replay reads each read job's blocks from the two files as its + /// worker would (`jobBlocks`). + fn run(platform: *const Platform, shape: Shape, options: Options) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const block_size = options.block_size; + var first_bytes: [96]u8 = undefined; + for (&first_bytes, 0..) |*byte, i| byte.* = @as(u8, @truncate(i *% 31 +% 7)) | 1; + var second_bytes: [128]u8 = undefined; + for (&second_bytes, 0..) |*byte, i| byte.* = @as(u8, @truncate(i *% 17 +% 3)) | 0x80; + // Rows of 16 bytes. a and b are adjacent in "first" and in the + // destination; c leaves a five-row gap before it and a two-row tail. + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "first", .name = "a", .shape = .init(.{ 3, 8 }, .u16), .offset = 4 }, + .{ .file_uri = "first", .name = "b", .shape = .init(.{ 2, 8 }, .u16), .offset = 52 }, + .{ .file_uri = "second", .name = "c", .shape = .init(.{ 4, 8 }, .u16), .offset = 0 }, + .{ .file_uri = "second", .name = "plain", .shape = .init(.{ 4, 8 }, .u16), .offset = 64 }, + }; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var paths: [2][std.Io.Dir.max_path_bytes]u8 = undefined; + var slots: [2]SourceSlot = undefined; + var opened: usize = 0; + defer for (slots[0..opened]) |*slot| slot.deinit(io); + for (&slots, &paths, [_][]const u8{ "first", "second" }, [_][]const u8{ &first_bytes, &second_bytes }) |*slot, *path, name, bytes| { + slot.* = .{ .uri = try writeTestFile(tmp.dir, io, name, bytes, path) }; + opened += 1; + } + var fixture: TestLoader = undefined; + try fixture.init(1); + defer fixture.deinit(); + const loader = &fixture.loader; + loader.plan_config = .{ .block_size = block_size, .request_size = options.request_size }; + var outputs: [2]Buffer = undefined; + var destinations: [2]Destination = undefined; + var parts = [_]Part{ + .{ .source = &sources[0], .source_slot = &slots[0], .destination = &destinations[0], .lattice = .{ .dst_offset = 0, .width = 48 } }, + .{ .source = &sources[1], .source_slot = &slots[0], .destination = &destinations[0], .lattice = .{ .dst_offset = 48, .width = 32 } }, + .{ .source = &sources[2], .source_slot = &slots[1], .destination = &destinations[0], .lattice = .{ .dst_offset = 160, .width = 64 } }, + .{ .source = &sources[3], .source_slot = &slots[1], .destination = &destinations[1], .lattice = .{ .width = 64 } }, + }; + const plain_shape = shape.setDim(0, 4); + destinations = .{ + .{ .shape = shape, .grid = .init(platform, shape), .output = &outputs[0], .parts = parts[0..3], .unread_bytes = .init(144) }, + .{ .shape = plain_shape, .grid = .init(platform, plain_shape), .output = &outputs[1], .parts = parts[3..4], .unread_bytes = .init(64) }, + }; + + const order = try Planner.sortedPartOrder(allocator, &parts); + defer allocator.free(order); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + var plans: [2]*Plan = undefined; + var plan_count: usize = 0; + var start: usize = 0; + while (start < order.len) : (plan_count += 1) { + const end = Planner.fileGroupEnd(&parts, order, start); + plans[plan_count] = try prepareTestPlan(arena.allocator(), &parts, order[start..end], .{ + .block_size = block_size, + .request_size = options.request_size, + }, 0); + start = end; + } + try std.testing.expectEqual(@as(usize, 2), plan_count); + // The fill goes out once, with the plan of the destination's first + // part, as its last job; one event per writer of every transfer. + try std.testing.expectEqual(.dma_fill, std.meta.activeTag(plans[0].jobs[plans[0].jobs.len - 1].kind)); + for (plans[0..plan_count], [_]usize{ 1, 0 }) |plan, fill_jobs| { + var writers: usize = 0; + var fills: usize = 0; + for (plan.jobs) |job| { + if (job.kind == .dma_fill) fills += 1; + for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| writers += @popCount(transfer.writer_mask); + } + try std.testing.expectEqual(fill_jobs, fills); + try std.testing.expectEqual(writers, plan.events.len); + } + + const zeros = try allocator.alloc(u8, 256); + defer allocator.free(zeros); + @memset(zeros, 0); + var replays: [2]Replay = undefined; + var initialized: usize = 0; + defer for (replays[0..initialized]) |*replay| replay.deinit(allocator); + for (&replays, &destinations) |*replay, *destination| { + replay.* = try .init(allocator, platform.placedShape(destination.shape).packedShape()); + initialized += 1; + } + var cross_part_piece = false; + for (plans[0..plan_count]) |plan| { + for (plan.jobs) |job| { + const fill = job.kind == .dma_fill; + const blocks = if (fill) &.{} else try jobBlocks(allocator, loader, plan, job); + defer allocator.free(blocks); + for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| { + // Every transfer stays inside one block (`Plan.Transfer`). + try std.testing.expect(transfer.landing % block_size + transfer.len <= block_size); + try std.testing.expect(transfer.landing < (if (fill) 1 else job.blockLen(block_size)) * block_size); + const bytes = if (fill) fill: { + try std.testing.expectEqual(@as(usize, 0), transfer.landing); + break :fill zeros[0..transfer.len]; + } else read: { + const file_offset = job.file_offset + transfer.landing; + // a's last byte and b's first one in the same piece. + if (plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; + break :read blocks[transfer.landing..][0..transfer.len]; + }; + const index: usize = if (transfer.destination == &destinations[0]) 0 else 1; + try replays[index].write(transfer.writer_mask, transfer.destination_offset, bytes); + } + } + } + try std.testing.expectEqual(options.cross_part_piece, cross_part_piece); + + var global: [256]u8 = @splat(0); + @memcpy(global[0..80], first_bytes[4..84]); + @memcpy(global[160..224], second_bytes[0..64]); + try replays[0].expect(allocator, &global); + try replays[1].expect(allocator, second_bytes[64..128]); + } + + /// A read job's blocks taken end to end once its worker has read it, + /// in file order. + fn jobBlocks(allocator: std.mem.Allocator, loader: *Loader, plan: *const Plan, job: Plan.Job) ![]u8 { + const block_size = loader.plan_config.block_size; + const bytes = try allocator.alloc(u8, job.blockLen(block_size) * block_size); + errdefer allocator.free(bytes); + @memset(bytes, 0xaa); + const read = job.kind.dma_read; + var buffers = [_][]u8{bytes[0..read.len]}; + try readSlot(loader, plan.source_slot, &buffers, job.file_offset, read.minimum_len); + return bytes; + } + + /// One destination's device buffers, written by replayed transfers. + const Replay = struct { + shape: Shape, + placement: Sharding.Placement, + writer_size: usize, + bytes: []u8, + written: []bool, + + fn init(allocator: std.mem.Allocator, placed_shape: Shape) !Replay { + const placement: Sharding.Placement = try .init(&placed_shape); + const writers = placement.mesh().?.devicesInCanonicalOrder().len; + const writer_size = placement.localShape(placed_shape).byteSize(); + const bytes = try allocator.alloc(u8, writers * writer_size); + errdefer allocator.free(bytes); + @memset(bytes, 0xaa); + const written = try allocator.alloc(bool, bytes.len); + @memset(written, false); + return .{ .shape = placed_shape, .placement = placement, .writer_size = writer_size, .bytes = bytes, .written = written }; + } + + fn deinit(self: *Replay, allocator: std.mem.Allocator) void { + allocator.free(self.written); + allocator.free(self.bytes); + } + + /// Each byte of each device at most once. + fn write(self: *Replay, writer_mask: u64, destination_offset: usize, source: []const u8) !void { + var mask = writer_mask; + while (mask != 0) { + const writer: usize = @intCast(@ctz(mask)); + mask &= mask - 1; + try std.testing.expect(destination_offset + source.len <= self.writer_size); + const at = writer * self.writer_size + destination_offset; + for (self.written[at..][0..source.len]) |*flag| { + try std.testing.expect(!flag.*); + flag.* = true; + } + @memcpy(self.bytes[at..][0..source.len], source); + } + } + + /// Every device holds its slice of the rank-2 `global`, entirely. + fn expect(self: *const Replay, allocator: std.mem.Allocator, global: []const u8) !void { + for (self.written) |flag| try std.testing.expect(flag); + const shape = self.shape; + const element_bytes = shape.dtype().sizeOf(); + const row_bytes: usize = @intCast(shape.dim(1) * element_bytes); + const expected = try allocator.alloc(u8, self.writer_size); + defer allocator.free(expected); + for (0..self.placement.mesh().?.devicesInCanonicalOrder().len) |writer| { + const slices = self.placement.slices(writer, shape); + const rows = slices.get(0); + const columns = slices.get(1); + const len: usize = @intCast(columns.size * element_bytes); + var cursor: usize = 0; + for (@intCast(rows.start)..@intCast(rows.start + rows.size)) |row| { + @memcpy(expected[cursor..][0..len], global[row * row_bytes + @as(usize, @intCast(columns.start * element_bytes)) ..][0..len]); + cursor += len; + } + try std.testing.expectEqualSlices(u8, expected, self.bytes[writer * self.writer_size ..][0..self.writer_size]); + } + } + }; +}; + +test "composite destinations place each part and zero-fill the rest of every device" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const shape = Shape.init(.{ .row = 16, .col = 8 }, .u16); + // Two by two devices, then two by four. + for ([_]u32{ 4, 8 }) |device_count| { + var platform = Platform.init(allocator, io, .cpu, .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = device_count }, + }) catch return error.SkipZigTest; + defer platform.deinit(allocator, io); + const physical = &platform.physical_mesh; + const logical: Sharding.LogicalMesh = .mesh(.{ .model = .high_bandwidth }); + const halves: Sharding.Mesh = try .init("halves", physical, logical, .parseBindings(.{ .model = .link_x })); + const quarters: Sharding.Mesh = if (device_count == 4) + try .init("quarters", physical, logical, .parseBindings(.{ .model = .{ .link_x, .link_y } })) + else + try .init("quarters", physical, logical, .parseBindings(.{ .model = .link_y })); + const all: Sharding.Mesh = try .init("all", physical, logical, .parseBindings(.{ .model = .{ .link_x, .link_y } })); + const scenarios = .{ + // Two row shards, each on half the devices: b and c straddle no + // boundary, the gap does. + .{ &halves, .{ .row = .model }, true }, + // Four row shards: b, c and the gap straddle shards. + .{ &quarters, .{ .row = .model }, true }, + // Every row of every source split across two devices, then + // across every device. + .{ &halves, .{ .col = .model }, false }, + .{ &all, .{ .col = .model }, false }, + // Every byte on every device: one span, so a and b merge. + .{ &all, .{}, true }, + }; + inline for (scenarios) |scenario| { + const placed = shape.withPartitioning(scenario[0], scenario[1]); + // 64-byte blocks hold a's tail and b's head together; 16-byte ones + // cut every row apart and every fill piece to one block. + try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .cross_part_piece = scenario[2] }); + try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 32, .cross_part_piece = false }); + } + } +} + // Scheduler tests. /// A plan of `job_count` unit jobs (`file_offset` = index), each with one diff --git a/zml/io/loader.zig b/zml/io/loader.zig index 8a0f19db..78b80cdb 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -13,10 +13,15 @@ //! rule), one part per piece: a part of the destination's logical bytes //! (the row-major bytes of its shape, before the sharding splits them), a //! run of `width` bytes repeated over strided dimensions. A whole -//! checkpoint tensor is one part. Each `LoadSpec` is one destination, which -//! the backends trust as `backend.LoadSpec` states. Only `Source` makes -//! bindings and its pieces are disjoint by construction: nothing here -//! checks that (Source's tests do). +//! checkpoint tensor is one part. +//! A strided part of at most `max_part_runs` runs, or of runs of at least +//! `large_run_bytes`, becomes one part per run; a spec with any other +//! strided part (row interleaves, transposes, broadcasts) is refused. The +//! parts go in destination order, and the bytes none covers are zero. Each +//! `LoadSpec` is one destination, which the backends trust as +//! `backend.LoadSpec` states. Only `Source` makes bindings and its pieces +//! are disjoint by construction: nothing here checks that (Source's tests +//! do). //! //! Order and admission. The tensors the loader fills go out first, as one //! submission. A group holds the tensors of consecutive programs in model @@ -46,6 +51,7 @@ const testing = @import("../testing.zig"); const backend = @import("backend.zig"); const Backend = backend.Backend; const buffered_loader = @import("buffered_loader.zig"); +const LoadPart = backend.LoadPart; const LoadSpec = backend.LoadSpec; const load_log = @import("log.zig").load; const TensorStore = @import("TensorStore.zig"); @@ -245,13 +251,13 @@ pub const Loader = struct { /// Every tensor and program input is checked once, before anything is /// published: a tensor `store` did not bind, a source that reads no /// bytes (an empty checkpoint tensor among them), a source that this - /// loader cannot assemble (a run that starts or ends inside a byte; - /// strided runs: a transpose, a broadcast, an interleave or interior - /// padding; or, on a platform that loads through the direct backend - /// (`backend.loadsDirect`), bytes no piece covers: padding), a sub-byte - /// last dimension that does not pack into whole bytes, a placement its - /// mesh does not divide or on devices the platform lacks, and a memory - /// kind the platform cannot take are `error.InvalidBinding`. + /// loader cannot assemble (a run that starts or ends inside a byte, or + /// strided runs that do not expand into contiguous ones: more than + /// `max_part_runs` runs, each shorter than `large_run_bytes`, as a + /// transpose, a broadcast or a row interleave makes), a sub-byte last + /// dimension that does not pack into whole bytes, a placement its mesh + /// does not divide or on devices the platform lacks, and a memory kind + /// the platform cannot take are `error.InvalidBinding`. /// /// `buffers` is `mem.bufferize` of `model`, whose buffers are empty. A /// tree of another structure (other union arms, optionals or slice @@ -735,6 +741,17 @@ fn prepareModel( return .{ .arena = arena, .specs = specs.items, .computed = computed.items }; } +/// A strided part with few runs, or with large ones, expands into one part +/// per run, each read and transferred on its own: at most `max_part_runs` +/// runs, or runs of at least `large_run_bytes`. A spec with any other +/// strided part is refused. +const max_part_runs = 64; +const large_run_bytes = 1 << 20; + +fn expandable(lattice: backend.Lattice) bool { + return lattice.dims.len == 0 or lattice.runs() <= max_part_runs or lattice.width >= large_run_bytes; +} + /// What a spec fills, as the logs name it: the model's tensor at `path` /// (`mem.Pair.path`) of `shape`, or `input` of the program computing it. const Target = struct { @@ -757,10 +774,10 @@ const Target = struct { /// The spec filling `output`, the buffer of `target`, from `source` into /// `memory`, its parts in `arena`: the source lowered to bytes -/// (`Source.Placed.lower`), in destination order. The one check of what -/// the backends trust (`backend.LoadSpec`); a rule it breaks is -/// `InvalidBinding`, logged with the target, and with the key its source -/// reads once the source is lowered. +/// (`Source.Placed.lower`), expanded into its runs, in destination order. +/// The one check of what the backends trust (`backend.LoadSpec`); a rule it +/// breaks is `InvalidBinding`, logged with the target, and with the key its +/// source reads once the source is lowered. fn prepareSpec( arena: std.mem.Allocator, platform: *const Platform, @@ -770,42 +787,51 @@ fn prepareSpec( output: *Buffer, ) PrepareError!LoadSpec { const shape = target.destination(); - const parts = source.lower(arena) catch |err| switch (err) { + const lowered = source.lower(arena) catch |err| switch (err) { error.OutOfMemory => |e| return e, error.SubBytePosition => { load_log.err("{f}: its source {f} reads a run that starts or ends inside a byte, which the loader cannot place", .{ target, source.shape }); return error.InvalidBinding; }, }; - if (parts.len == 0) { + if (lowered.len == 0) { load_log.err("{f} reads no checkpoint bytes", .{target}); return error.InvalidBinding; } - const key = parts[0].source.name; + const key = lowered[0].source.name; try checkPlacement(platform, target, key); if (!backend.supportsMemory(platform, memory)) { load_log.err("{f} from {s} is bound to {t} memory, which {t} cannot take", .{ target, key, memory, platform.target }); return error.InvalidBinding; } - for (parts) |part| { - if (part.lattice.dims.len != 0) { - load_log.err("{f} from {s}: a part of {d} strided runs of {d} bytes (a transpose, a broadcast, an interleave or interior padding), which this loader does not assemble", .{ target, part.source.name, part.lattice.runs(), part.lattice.width }); + var runs: u64 = 0; + for (lowered) |part| { + if (!expandable(part.lattice)) { + load_log.err("{f} from {s}: a part of {d} strided runs of {d} bytes (a transpose, a broadcast or an interleave), more than {d} runs under {d} bytes, which this loader does not assemble", .{ target, part.source.name, part.lattice.runs(), part.lattice.width, max_part_runs, large_run_bytes }); return error.InvalidBinding; } + runs += part.lattice.runs(); } + const parts = if (runs == lowered.len) lowered else expanded: { + const expanded = try arena.alloc(LoadPart, @intCast(runs)); + var next: usize = 0; + for (lowered) |part| { + var cursor: backend.Lattice.Cursor = .init(part.lattice, 0, part.lattice.runs()); + while (cursor.next()) |run| : (next += 1) { + expanded[next] = .{ .source = part.source, .lattice = .{ .src_offset = run.src, .dst_offset = run.dst, .width = part.lattice.width } }; + } + } + break :expanded expanded; + }; + std.mem.sort(LoadPart, parts, {}, struct { + fn lessThan(_: void, lhs: LoadPart, rhs: LoadPart) bool { + return lhs.lattice.dst_offset < rhs.lattice.dst_offset; + } + }.lessThan); if (!backend.checkParts(parts, shape.byteSize())) { load_log.err("{f} from {s}: its parts leave their sources or its {d} bytes, pass the largest file offset, or come out of order", .{ target, key, shape.byteSize() }); return error.InvalidBinding; } - // The direct backend writes the bytes its parts cover, and no others. - if (backend.loadsDirect(platform)) { - var covered: u64 = 0; - for (parts) |part| covered += part.lattice.width; - if (covered != shape.byteSize()) { - load_log.err("{f} from {s}: {d} of its {d} bytes covered by no part (padding), which this loader does not zero on {t}", .{ target, key, shape.byteSize() - covered, shape.byteSize(), platform.target }); - return error.InvalidBinding; - } - } return .{ .shape = shape, .memory = memory, .output = output, .parts = parts }; } @@ -1407,9 +1433,9 @@ test "a rejected binding or a program of the wrong output fails the loader" { const splits = platform.devices.len > 1 and 4 % platform.devices.len == 0; // No binding, an empty source or program input, a mesh of other - // devices, a host kind the platform cannot take, a broadcast's strided - // runs, padding where the direct backend loads, and a program of the - // wrong shape or placement. + // devices, a host kind the platform cannot take, a broadcast of more + // strided runs than expand, and a program of the wrong shape or + // placement. const Case = struct { tensor: Tensor, at_wait: bool = false, skip: bool = false }; const cases = [_]Case{ .{ .tensor = Tensor.fromShape(fixture.value.shape()) }, @@ -1419,8 +1445,7 @@ test "a rejected binding or a program of the wrong output fails the loader" { .{ .tensor = foreign[1] }, .{ .tensor = fixture.computed("value", foreign[2], LoaderTestFixture.Identity{}) }, .{ .tensor = pinned, .skip = backend.supportsMemory(platform, .host_pinned) }, - .{ .tensor = fixture.store.view().source("value", .{.x}).broad(Shape.init(.{ .y = 2, .x = 4 }, .u8)).withPartitioning(.replicated, .{}).bind() }, - .{ .tensor = fixture.store.view().source("second", .{.x}).pad(0, .{ .x = TensorStore.Source.Pad{ .high = 2 } }).withPartitioning(.replicated, .{}).bind(), .skip = !backend.loadsDirect(platform) }, + .{ .tensor = fixture.store.view().source("value", .{.x}).broad(Shape.init(.{ .y = max_part_runs + 1, .x = 4 }, .u8)).withPartitioning(.replicated, .{}).bind() }, .{ .tensor = fixture.computed("value", fixture.value, Rows{}), .at_wait = true }, .{ .tensor = fixture.computed("value", misplaced, LoaderTestFixture.Identity{}), .at_wait = true, .skip = !splits }, }; @@ -1492,6 +1517,293 @@ fn expectStructureMismatch(fixture: *LoaderTestFixture, comptime Model: type, bu try std.testing.expectEqual(0, written); } +/// Two safetensors files for composite loads, every row 16 bytes (8 u16): +/// `first` holds `a` [3, 8] then `b` [2, 8], adjacent, then `spacer`, then +/// `q` [8, 8], `k` [4, 8] and `v` [4, 8]; `second` holds `plain` [4, 8], +/// then `c` [4, 8] and the experts `e{0..3}.w1` and `e{0..3}.w3` [2, 8]. +/// Pinned after `init`. Each store takes a copy of `registry` +/// (`testing.storeOf`). +const CompositeFixture = struct { + tmp: std.testing.TmpDir, + registry: safetensors.TensorRegistry, + a: [48]u8, + b: [32]u8, + c: [64]u8, + plain: [64]u8, + q: [128]u8, + k: [64]u8, + v: [64]u8, + w1: [4][32]u8, + w3: [4][32]u8, + + const expert_names = [2][4][]const u8{ + .{ "e0.w1", "e1.w1", "e2.w1", "e3.w1" }, + .{ "e0.w3", "e1.w3", "e2.w3", "e3.w3" }, + }; + + fn init(self: *CompositeFixture, allocator: std.mem.Allocator, io: std.Io) !void { + self.tmp = std.testing.tmpDir(.{}); + errdefer self.tmp.cleanup(); + self.registry = .init(allocator); + errdefer self.registry.deinit(); + testing.pattern(&self.a, 1); + testing.pattern(&self.b, 2); + testing.pattern(&self.c, 3); + testing.pattern(&self.plain, 4); + testing.pattern(&self.q, 9); + testing.pattern(&self.k, 10); + testing.pattern(&self.v, 11); + for (&self.w1, &self.w3, 0..) |*w1, *w3, e| { + testing.pattern(w1, @intCast(20 + e)); + testing.pattern(w3, @intCast(30 + e)); + } + var spacer: [16]u8 = undefined; + testing.pattern(&spacer, 5); + try testing.writeSafetensors(allocator, io, self.tmp.dir, "first.safetensors", &self.registry, &.{ + .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &self.a }, + .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.b }, + .{ .name = "spacer", .dtype = "U16", .dims = &.{ 1, 8 }, .bytes = &spacer }, + .{ .name = "q", .dtype = "U16", .dims = &.{ 8, 8 }, .bytes = &self.q }, + .{ .name = "k", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.k }, + .{ .name = "v", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.v }, + }); + try testing.writeSafetensors(allocator, io, self.tmp.dir, "second.safetensors", &self.registry, &.{ + .{ .name = "plain", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.plain }, + .{ .name = "c", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.c }, + .{ .name = expert_names[0][0], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[0] }, + .{ .name = expert_names[1][0], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[0] }, + .{ .name = expert_names[0][1], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[1] }, + .{ .name = expert_names[1][1], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[1] }, + .{ .name = expert_names[0][2], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[2] }, + .{ .name = expert_names[1][2], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[2] }, + .{ .name = expert_names[0][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[3] }, + .{ .name = expert_names[1][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[3] }, + }); + } + + fn deinit(self: *CompositeFixture) void { + self.registry.deinit(); + self.tmp.cleanup(); + } + + fn rows(view: TensorStore.View, name: []const u8) TensorStore.Source { + return view.source(name, .{ .row, .col }); + } +}; + +test "loader assembles concatenated and padded sources on every placement and memory kind" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // a, b and c end to end over 16 rows; c alone over 8 rows; b then a + // over 6 rows. The rest is zero. + var concatenated: [256]u8 = @splat(0); + @memcpy(concatenated[0..48], &fixture.a); + @memcpy(concatenated[48..80], &fixture.b); + @memcpy(concatenated[80..144], &fixture.c); + var padded: [128]u8 = @splat(0); + @memcpy(padded[0..64], &fixture.c); + var reversed: [96]u8 = @splat(0); + @memcpy(reversed[0..32], &fixture.b); + @memcpy(reversed[32..80], &fixture.a); + + const Source = TensorStore.Source; + const rows = CompositeFixture.rows; + const Model = struct { concatenated: Tensor, padded: Tensor, reversed: Tensor, plain: Tensor }; + for ([_]usize{ 2, 4 }) |device_count| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const platform = devices.platform; + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + // Rows over the devices (b and c straddle shards on four), every + // row split over them, and every byte on each of them. + inline for (.{ .{ .row = .model }, .{ .col = .model }, .{} }) |partitioning| { + for ([_]Memory{ .default, .host_pinned, .host_mapped }) |memory| { + if (!backend.supportsMemory(platform, memory)) continue; + const view = store.view(); + const model: Model = .{ + .concatenated = Source.concatenate(&.{ rows(view, "a"), rows(view, "b"), rows(view, "c") }, .row) + .pad(0, .{ .row = Source.Pad{ .high = 7 } }) + .withPartitioning(.tp, partitioning).bind(), + // The padded engram-style table: one source. + .padded = rows(view, "c").pad(0, .{ .row = Source.Pad{ .high = 4 } }).withPartitioning(.tp, partitioning).bind(), + // Out of file order, on the platform's replicated mesh. + .reversed = Source.concatenate(&.{ rows(view, "b"), rows(view, "a") }, .row) + .pad(0, .{ .row = Source.Pad{ .high = 1 } }) + .withPartitioning(.replicated, .{}).bind(), + .plain = rows(view, "plain").withPartitioning(.tp, partitioning).bind(), + }; + inline for (comptime std.meta.fieldNames(Model)) |name| view.setMemory(@field(model, name), memory); + try std.testing.expect(store.bindingOf(model.plain.id).?.load.source.whole() != null); + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + try testing.expectShards(allocator, io, &buffers.concatenated, &concatenated, memory); + try testing.expectShards(allocator, io, &buffers.padded, &padded, memory); + try testing.expectShards(allocator, io, &buffers.reversed, &reversed, memory); + try testing.expectShards(allocator, io, &buffers.plain, &fixture.plain, memory); + // Each destination once, padding included. + try std.testing.expectEqual(@as(usize, 256 + 128 + 96 + 64), loader.stats.bytes_loaded); + } + } + } +} + +test "loader assembles interleaved, stacked and split sources" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const Source = TensorStore.Source; + const rows = CompositeFixture.rows; + const row = 16; + + // Experts with their two projections concatenated, then row by row. + var concatenated_experts: [256]u8 = undefined; + var interleaved_experts: [256]u8 = undefined; + for (0..4) |e| { + @memcpy(concatenated_experts[e * 64 ..][0..32], &fixture.w1[e]); + @memcpy(concatenated_experts[e * 64 + 32 ..][0..32], &fixture.w3[e]); + for (0..2) |r| { + @memcpy(interleaved_experts[e * 64 + r * 2 * row ..][0..row], fixture.w1[e][r * row ..][0..row]); + @memcpy(interleaved_experts[e * 64 + (r * 2 + 1) * row ..][0..row], fixture.w3[e][r * row ..][0..row]); + } + } + + const Model = struct { qkv: Tensor, concatenated: Tensor, interleaved: Tensor, head: Tensor, tail: Tensor }; + for ([_]usize{ 2, 4 }) |device_count| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const platform = devices.platform; + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + const view = store.view(); + const groups: i64 = @intCast(device_count); + + // q, k and v cut in one group per device: each device holds its rows + // of every member, contiguous. Every part has `groups` runs. + var members: [3]Source = undefined; + for (&members, [_][]const u8{ "q", "k", "v" }) |*member, name| { + member.* = rows(view, name).splitAxis(.row, .{ .group = groups, .r = -1 }); + } + var experts: [2]Source = undefined; + for (&experts, CompositeFixture.expert_names) |*stacked, names| { + var per_expert: [4]Source = undefined; + for (&per_expert, names) |*expert, name| expert.* = rows(view, name); + stacked.* = Source.stack(&per_expert, 0, .expert); + } + const fused = Source.concatenate(&.{ rows(view, "q"), rows(view, "k") }, .row); + // A part of q, and the rest of q with all of k. + const head, const tail = fused.split(.row, &.{ 4, 8 })[0..2].*; + const model: Model = .{ + .qkv = Source.concatenate(&members, .r).merge(.{ .row = .{ .group, .r } }).withPartitioning(.tp, .{ .row = .model }).bind(), + // Experts are whole rows of the merged axis, one or two per device. + .concatenated = Source.concatenate(&experts, .row) + .merge(.{ .rows = .{ .expert, .row } }) + .withPartitioning(.tp, .{ .rows = .model }).bind(), + .interleaved = Source.stack(&experts, 2, .proj) + .merge(.{ .rows = .{ .expert, .row, .proj } }) + .withPartitioning(.tp, .{ .rows = .model }).bind(), + .head = head.withPartitioning(.replicated, .{}).bind(), + .tail = tail.withPartitioning(.tp, .{ .row = .model }).bind(), + }; + var lowering: std.heap.ArenaAllocator = .init(allocator); + defer lowering.deinit(); + const qkv_parts = try store.bindingOf(model.qkv.id).?.load.source.lower(lowering.allocator()); + try std.testing.expectEqual(@as(usize, 3), qkv_parts.len); + for (qkv_parts) |part| try std.testing.expectEqual(@as(u64, device_count), part.lattice.runs()); + + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + var qkv: [256]u8 = undefined; + var at: usize = 0; + for (0..device_count) |g| { + for ([_][]const u8{ &fixture.q, &fixture.k, &fixture.v }) |member| { + const per_group = member.len / device_count; + @memcpy(qkv[at..][0..per_group], member[g * per_group ..][0..per_group]); + at += per_group; + } + } + try testing.expectShards(allocator, io, &buffers.qkv, &qkv, .default); + try testing.expectShards(allocator, io, &buffers.concatenated, &concatenated_experts, .default); + try testing.expectShards(allocator, io, &buffers.interleaved, &interleaved_experts, .default); + try testing.expectShards(allocator, io, &buffers.head, fixture.q[0..64], .default); + var tail_bytes: [128]u8 = undefined; + @memcpy(tail_bytes[0..64], fixture.q[64..128]); + @memcpy(tail_bytes[64..], &fixture.k); + try testing.expectShards(allocator, io, &buffers.tail, &tail_bytes, .default); + } +} + +test "loader fills gaps, reordered and strided rows" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const platform = testing.env(); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + const Source = TensorStore.Source; + const Pad = Source.Pad; + const view = store.view(); + const rows = CompositeFixture.rows; + const a = rows(view, "a"); + + // b's second row past a gap, the rest zero; b then a, each padded; a's + // last row before its first two, one source feeding two parts; and a's + // rows every other row, a strided part of few runs, expanded. + const Model = struct { sparse: Tensor, padded: Tensor, swapped: Tensor, strided: Tensor }; + const model: Model = .{ + .sparse = rows(view, "b").slice(.row, .{ .start = 1 }).pad(0, .{ .row = Pad{ .low = 4, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), + .padded = Source.concatenate(&.{ + rows(view, "b").pad(0, .{ .row = Pad{ .high = 2 } }), + a.pad(0, .{ .row = Pad{ .high = 1 } }), + }, .row).withPartitioning(.replicated, .{}).bind(), + .swapped = Source.concatenate(&.{ a.slice(.row, .{ .start = 2 }), a.slice(.row, .{ .end = 2 }) }, .row) + .pad(0, .{ .row = Pad{ .high = 5 } }).withPartitioning(.replicated, .{}).bind(), + .strided = a.pad(0, .{ .row = Pad{ .interior = 1, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + var expected: [128]u8 = @splat(0); + @memcpy(expected[64..80], fixture.b[16..32]); + try testing.expectShards(allocator, io, &buffers.sparse, &expected, .default); + expected = @splat(0); + @memcpy(expected[0..32], &fixture.b); + @memcpy(expected[64..112], &fixture.a); + try testing.expectShards(allocator, io, &buffers.padded, &expected, .default); + expected = @splat(0); + @memcpy(expected[0..16], fixture.a[32..48]); + @memcpy(expected[16..48], fixture.a[0..32]); + try testing.expectShards(allocator, io, &buffers.swapped, &expected, .default); + expected = @splat(0); + for (0..3) |i| @memcpy(expected[i * 32 ..][0..16], fixture.a[i * 16 ..][0..16]); + try testing.expectShards(allocator, io, &buffers.strided, &expected, .default); +} + test "a program takes its inputs from sources, in order" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -1557,16 +1869,18 @@ test "the front end loads through the buffered backend and fails with it" { defer loader.deinit(); // A tensor, one assembled from value's first half and second's last - // half, and a program tensor, in a group of its own. + // half, one padded with zeros, and a program tensor, in a group of its + // own. const Source = TensorStore.Source; const view = fixture.store.view(); - const Model = struct { value: Tensor, assembled: Tensor, computed: Tensor }; + const Model = struct { value: Tensor, assembled: Tensor, padded: Tensor, computed: Tensor }; const model: Model = .{ .value = fixture.value, .assembled = Source.concatenate(&.{ view.source("value", .{.x}).slice(.x, .{ .end = 2 }), view.source("second", .{.x}).slice(.x, .{ .start = 2 }), }, .x).withPartitioning(.replicated, .{}).bind(), + .padded = view.source("second", .{.x}).pad(0, .{ .x = Source.Pad{ .high = 2 } }).withPartitioning(.replicated, .{}).bind(), .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Add{ .value = 1 }), }; var buffers = try mem.bufferize(allocator, &model); @@ -1576,6 +1890,7 @@ test "the front end loads through the buffered backend and fails with it" { try loader.wait(); try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); try testing.expectShards(allocator, io, &buffers.assembled, &.{ 1, 2, 7, 8 }, .default); + try testing.expectShards(allocator, io, &buffers.padded, &.{ 5, 6, 7, 8, 0, 0 }, .default); try testing.expectShards(allocator, io, &buffers.computed, &.{ 2, 3, 4, 5 }, .default); // A missing file fails the loader, and every later call with it. From be272ceeb0530b4f15ea59e4c102b7234b5bb07f Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 15:29:58 +0100 Subject: [PATCH 083/103] zml/io: stage column-sharded tensors by destination box A read job of a tensor sharded along any axis but its first went to the devices in file order, one DMA piece per row and device: Llama-3.1-8B with o_proj and down_proj column-sharded over four CPU devices planned 1.06M submissions. The planner now lists a job's segments before it places them, and stages the job where that saves at least one device's in-flight pieces (64) and the segments tile its bytes once: the bytes land in the job's blocks grouped by destination box, so a box goes out as one piece per block. The same load plans about 10,500 submissions. A staged job reads one extent. An exact read scatters straight into the blocks, at most one vectored read's ranges per call; a read widened for direct I/O lands in the worker's page-aligned bounce buffer and is copied from there. The batch planning debug line counts staged jobs. The composite replays also run staged, with reads exact and widened, and a new test loads column-sharded tensors over four devices in under a hundredth of their row pieces. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 542 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 519 insertions(+), 23 deletions(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 6f4660b4..8b8239b0 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -47,7 +47,9 @@ //! writers and both offsets are contiguous, and its parts may come from //! different files and plans. Bytes no part covers go out with the plan of //! its first part, as a last fill job that reads nothing and transfers from -//! the zero block, which no credit owns. +//! the zero block, which no credit owns. The job of a tensor sharded along +//! any axis but its first is staged by destination where that saves +//! submissions (`JobCut.placeDma`). //! //! Locks. Where several pumps are held, they are taken in device order. A //! pump's mutex may be held while a released request gives its credit back @@ -425,7 +427,7 @@ pub const Loader = struct { } fn workerMain(self: *Loader) void { - var scratch = ReadRequest.Scratch.init(self.allocator, self.credits.blocks_per_slot) catch |err| { + var scratch = ReadRequest.Scratch.init(self.allocator, self.credits.blocks_per_slot, self.plan_config.block_size) catch |err| { self.recordError(err, "allocate source worker scratch: blocks={d}", .{self.credits.blocks_per_slot}); return; }; @@ -529,7 +531,7 @@ pub const Loader = struct { else @as(f64, @floatFromInt(batch.parts.len)) / @as(f64, @floatFromInt(source_jobs)); - load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ + load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, staged_jobs={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ batch.sequence, batch.plans.items.len, @as(f64, @floatFromInt(planning_ns)) / std.time.ns_per_s, @@ -539,6 +541,7 @@ pub const Loader = struct { batch.parts.len, transfers, dma_submissions, + kinds.get(.dma_land), coalescing_ratio, average_read_size, }); @@ -746,6 +749,10 @@ const Plan = struct { /// Read into pinned blocks in file order, where its transfers /// address it, then copied to the devices by DMA. dma_read: Read, + /// Reads its extents, whose items land their bytes in its + /// blocks: a read of its plan's file grouped by destination + /// box (`Planner.JobCut.stage`). + dma_land: Land, /// Reads nothing: every transfer goes from the loader's zero /// block (`Planner.JobCut.appendFillTransfers`). dma_fill, @@ -759,16 +766,60 @@ const Plan = struct { minimum_len: usize, }; - /// The blocks of its credit's slot the job reads into; none for a - /// fill job. + /// A landing job: `len` bytes in its blocks, landed by the + /// `extent_len` extents of the plan's `extents` from + /// `extent_start`, in order. + const Land = struct { + len: usize, + extent_start: usize, + extent_len: usize, + }; + + /// The blocks of its credit's slot the job reads into or lands + /// in; none for a fill job. fn blockLen(self: Job, block_size: usize) usize { return switch (self.kind) { .dma_read => |read| Planner.blockCount(read.len, block_size), + .dma_land => |land| Planner.blockCount(land.len, block_size), .dma_fill => 0, }; } }; + /// Bytes an extent of a landing job places in its blocks: the runs + /// `first` to `end` of `lattice`, its source offsets in the file + /// and its destination offsets logical, clipped to the logical + /// range `[clip_start, clip_end)`. A byte at logical offset `l`, + /// which `grid.locate(l)` puts at offset `o` of box `b` of the + /// destination's grid, lands at `landings[landing_start + b] +% o` + /// of the job's blocks taken end to end (`walkItem`). + const Item = struct { + destination: *Destination, + lattice: backend.Lattice, + first: u64 = 0, + end: u64 = 1, + clip_start: u64, + clip_end: u64, + landing_start: usize, + }; + + /// One read of a landing job: `len` bytes from `file_offset` of + /// `source_slot`'s file, of which the first `minimum_len` exist, + /// landed by the `item_len` items of the plan's `items` from + /// `item_start`. + const Extent = struct { + source_slot: *SourceSlot, + file_offset: u64, + len: usize, + minimum_len: usize, + /// Read straight into the job's blocks: an exact read whose + /// items' runs tile it, in order. Otherwise read into the + /// worker's bounce buffer and copied from there. + scatter: bool, + item_start: usize, + item_len: usize, + }; + /// `len` bytes at `landing` of the job's blocks taken end to end, for /// every device of `writer_mask` at `destination_offset` of its shard. /// Every transfer stays inside one block. A fill one lands at 0 and @@ -788,6 +839,11 @@ const Plan = struct { source_slot: *SourceSlot, jobs: []Job, transfers: []Transfer, + /// The landing jobs' extents, their items and the items' landing + /// tables (one entry per box of the item's destination). + extents: []Extent = &.{}, + items: []Item = &.{}, + landings: []u64 = &.{}, requests: []ReadRequest, events: []Dma.EventContext, /// Event slots handed out so far, one `fetchAdd` per submission from @@ -820,6 +876,9 @@ const Planner = struct { /// The longest read, a whole number of blocks /// (`backend.requestSize`). request_size: usize, + /// Submissions a DMA job must save to be staged (`JobCut.stage`): + /// one device's in-flight pieces. + minimum_staging_gain: usize = max_dma_pieces_per_device, }; /// How each file is read: aligned when it is local and the VFS that @@ -935,6 +994,11 @@ const Planner = struct { /// One part's bytes in a read job: the logical range `[start, end)` /// of its destination, from `file_offset` of the file on. const Segment = struct { destination: *Destination, start: u64, end: u64, file_offset: u64 }; + /// A segment's bytes on box `box` of its destination: `len` from + /// `box_offset`, and where a staged job lands them. + const Entry = struct { segment: u32, box: u32, group: u32 = 0, box_offset: u64, len: u64, landing: u64 = 0 }; + /// What a staged job lands together. + const Group = struct { destination: *Destination, box: u32 }; allocator: std.mem.Allocator, parts: []Part, @@ -946,7 +1010,15 @@ const Planner = struct { maximum_job_len: usize, jobs: std.ArrayList(Plan.Job) = .empty, transfers: std.ArrayList(Plan.Transfer) = .empty, + items: std.ArrayList(Plan.Item) = .empty, + landings: std.ArrayList(u64) = .empty, + extents: std.ArrayList(Plan.Extent) = .empty, safe_boundaries: std.ArrayList(u64) = .empty, + /// A read job's segments before they are placed, and the staging + /// scratch. + segments: std.ArrayList(Segment) = .empty, + entries: std.ArrayList(Entry) = .empty, + groups: std.AutoHashMapUnmanaged(Group, u32) = .empty, source_bytes: u64 = 0, /// The plan of the jobs cut, copied into `arena` with every context @@ -963,6 +1035,9 @@ const Planner = struct { .source_slot = self.source_slot, .jobs = try arena.dupe(Plan.Job, jobs), .transfers = try arena.dupe(Plan.Transfer, transfers), + .extents = try arena.dupe(Plan.Extent, self.extents.items), + .items = try arena.dupe(Plan.Item, self.items.items), + .landings = try arena.dupe(u64, self.landings.items), .requests = requests, .events = try arena.alloc(Dma.EventContext, submissionCount(transfers)), .source_bytes = self.source_bytes, @@ -972,7 +1047,13 @@ const Planner = struct { fn deinit(self: *JobCut) void { const allocator = self.allocator; + self.groups.deinit(allocator); + self.entries.deinit(allocator); + self.segments.deinit(allocator); self.safe_boundaries.deinit(allocator); + self.extents.deinit(allocator); + self.landings.deinit(allocator); + self.items.deinit(allocator); self.transfers.deinit(allocator); self.jobs.deinit(allocator); } @@ -984,7 +1065,8 @@ const Planner = struct { /// as its block is read. With an alignment (`Reads.alignmentOf`), a /// job's read widens to aligned bounds around its range /// (`maximumJobLen` leaves room for it), and the transfers address - /// the widened read. + /// the widened read. Each job's segments are placed in its blocks + /// by `placeDma`. fn cutRuns(self: *JobCut, order: []const usize) !void { const allocator = self.allocator; const parts = self.parts; @@ -1057,6 +1139,9 @@ const Planner = struct { const read_end = if (alignment == 0) job_end else std.mem.alignForward(u64, job_end, alignment); const read_len: usize = @intCast(read_end - read_start); const transfer_start = self.transfers.items.len; + // The job's segments first: where they land in its + // blocks depends on all of them. + self.segments.clearRetainingCapacity(); while (candidate_start < run_part_end) { const candidate = parts[order[candidate_start]].fileRange(); if (candidate.end > job_start) break; @@ -1070,19 +1155,21 @@ const Planner = struct { const intersection_end = @min(job_end, range.end); if (intersection_start >= intersection_end) continue; const logical_start = part.lattice.dst_offset + (intersection_start - range.start); - try self.appendRunTransfers(transfer_start, .{ + try self.segments.append(allocator, .{ .destination = part.destination, .start = logical_start, .end = logical_start + (intersection_end - intersection_start), .file_offset = intersection_start, - }, read_start); + }); } - std.debug.assert(self.transfers.items.len != transfer_start); + std.debug.assert(self.segments.items.len != 0); + const read: Plan.Job.Read = .{ .len = read_len, .minimum_len = @intCast(job_end - read_start) }; + const kind = try self.placeDma(read, read_start, job_start, job_len); try self.jobs.append(allocator, .{ .file_offset = read_start, .transfer_start = transfer_start, .transfer_len = self.transfers.items.len - transfer_start, - .kind = .{ .dma_read = .{ .len = read_len, .minimum_len = @intCast(job_end - read_start) } }, + .kind = kind, }); self.source_bytes +|= @intCast(job_len); job_start = job_end; @@ -1093,6 +1180,150 @@ const Planner = struct { } } + /// Places a DMA job's segments (`segments`, in file order) in its + /// blocks. Blocks holding the read in file order give one transfer + /// per run (`ShardGrid.run`): a tensor sharded along any axis but + /// its first takes one per row and device, as a column-sharded + /// `o_proj` or `down_proj` does. A staged job lands its read grouped + /// by destination box instead (`stage`), so a box takes one piece + /// per block. Llama-3.1-8B sharded like that on four B70, page + /// cache warm, planned 1.05M submissions and loaded in 5.1 s (~1.2 + /// ms per piece, 64 in flight per device); staged, 5,468 and 0.71 + /// s, against 0.67 s sharded by rows. Staging costs a read + /// scattered over the runs, or for a direct read a copy through the + /// worker's bounce buffer (`landJob`), so a job is staged only where + /// it saves `Config.minimum_staging_gain` submissions, and only when + /// its segments tile its bytes once: overlapping parts send a source + /// byte to two segments, and one read cannot land it twice. + fn placeDma(self: *JobCut, read: Plan.Job.Read, read_start: u64, data_start: u64, job_len: usize) !Plan.Job.Kind { + const block_size = self.config.block_size; + const transfer_start = self.transfers.items.len; + // File order takes a submission per writer of every run, cut by + // blocks, less where runs of adjacent segments continue each other + // or a run ends where a block does: at most `slack` fewer. + var in_file_order: usize = 0; + var slack: usize = 0; + var tiles = true; + var next = data_start; + for (self.segments.items) |segment| { + if (segment.file_offset != next) tiles = false; + next = segment.file_offset + (segment.end - segment.start); + const grid = &segment.destination.grid; + const landing = segment.file_offset - read_start; + const crossings = (landing + segment.end - segment.start - 1) / block_size - landing / block_size; + in_file_order += (grid.runCount(segment.start, segment.end) + crossings) * grid.replicas(); + slack += (1 + crossings) * grid.replicas(); + } + if (tiles and next == data_start + job_len) { + if (try self.stage(read, read_start, job_len, in_file_order, slack)) |land| return .{ .dma_land = land }; + } + for (self.segments.items) |segment| try self.appendRunTransfers(transfer_start, segment, read_start); + return .{ .dma_read = read }; + } + + /// Lands the job's segments box by box: their bytes on each box of a + /// destination, grouped by (destination, box) in order of first + /// appearance, each group in box order. The job's read is its one + /// extent, and one `Item` per segment tells the worker where its bytes + /// go (`landJob`). Null, with nothing appended, when that saves + /// fewer than `Config.minimum_staging_gain` submissions against file + /// order, estimated at `in_file_order` and at most `slack` fewer: + /// counted exactly, by listing the runs, only when that decides. + fn stage(self: *JobCut, read: Plan.Job.Read, read_start: u64, job_len: usize, in_file_order: usize, slack: usize) !?Plan.Job.Land { + const allocator = self.allocator; + const segments = self.segments.items; + self.entries.clearRetainingCapacity(); + self.groups.clearRetainingCapacity(); + for (segments, 0..) |segment, index| { + const grid = &segment.destination.grid; + for (0..grid.box_count) |box| { + const low = grid.rankIn(box, segment.start); + const high = grid.rankIn(box, segment.end); + if (high == low) continue; + const group = try self.groups.getOrPut(allocator, .{ .destination = segment.destination, .box = @intCast(box) }); + if (!group.found_existing) group.value_ptr.* = @intCast(self.groups.count() - 1); + try self.entries.append(allocator, .{ .segment = @intCast(index), .box = @intCast(box), .group = group.value_ptr.*, .box_offset = low, .len = high - low }); + } + } + const entries = self.entries.items; + std.mem.sort(Entry, entries, {}, struct { + fn lessThan(_: void, lhs: Entry, rhs: Entry) bool { + if (lhs.group != rhs.group) return lhs.group < rhs.group; + if (lhs.box_offset != rhs.box_offset) return lhs.box_offset < rhs.box_offset; + return lhs.segment < rhs.segment; + } + }.lessThan); + // The read's own order when every box of a segment is one run and + // the groups come in the order of the runs. + var runs: u64 = 0; + for (segments) |segment| runs += segment.destination.grid.runCount(segment.start, segment.end); + if (runs == entries.len) { + const in_order = for (entries[1..], entries[0 .. entries.len - 1]) |entry, previous| { + if (entry.segment < previous.segment or (entry.segment == previous.segment and entry.box < previous.box)) break false; + } else true; + if (in_order) return null; + } + const staged_start = self.transfers.items.len; + var landing: u64 = 0; + for (entries) |*entry| { + const destination = segments[entry.segment].destination; + entry.landing = landing; + try self.appendBlocked(staged_start, .{ + .destination = destination, + .landing = @intCast(landing), + .writer_mask = destination.grid.writers[entry.box], + .destination_offset = @intCast(entry.box_offset), + .len = @intCast(entry.len), + }); + landing += entry.len; + } + std.debug.assert(landing == job_len); + const needed = submissionCount(self.transfers.items[staged_start..]) +| self.config.minimum_staging_gain; + var file_order_wins = needed > in_file_order; + if (!file_order_wins and needed +| slack > in_file_order) { + const exact_start = self.transfers.items.len; + for (segments) |segment| try self.appendRunTransfers(exact_start, segment, read_start); + file_order_wins = needed > submissionCount(self.transfers.items[exact_start..]); + self.transfers.shrinkRetainingCapacity(exact_start); + } + if (file_order_wins) { + self.transfers.shrinkRetainingCapacity(staged_start); + return null; + } + + // A landing table per segment, one entry per box of its + // destination. + const item_start = self.items.items.len; + for (segments) |segment| { + const landing_start = self.landings.items.len; + try self.landings.appendNTimes(allocator, 0, segment.destination.grid.box_count); + try self.items.append(allocator, .{ + .destination = segment.destination, + .lattice = .{ .src_offset = segment.file_offset, .dst_offset = segment.start, .width = segment.end - segment.start }, + .clip_start = segment.start, + .clip_end = segment.end, + .landing_start = landing_start, + }); + } + for (entries) |entry| { + const item = self.items.items[item_start + entry.segment]; + self.landings.items[item.landing_start + entry.box] = entry.landing -% entry.box_offset; + } + // Only a direct read has padding around its tensor bytes, which + // goes through the worker's bounce buffer. + const extent_start = self.extents.items.len; + try self.extents.append(allocator, .{ + .source_slot = self.source_slot, + .file_offset = read_start, + .len = read.len, + .minimum_len = read.minimum_len, + .scatter = self.alignment == 0, + .item_start = item_start, + .item_len = segments.len, + }); + return .{ .len = job_len, .extent_start = extent_start, .extent_len = 1 }; + } + /// Appends `segment`'s transfers with its bytes where the read holds /// them, one per run (`ShardGrid.run`) and block, merged with the /// previous transfer from `transfer_start` on where block, writers @@ -1386,17 +1617,36 @@ const Scheduler = struct { const ReadRequest = struct { const Scratch = struct { allocator: std.mem.Allocator, - /// A read's ranges: one per block of a job (`readBlocks`). + /// A read's ranges: one per block of a job (`readBlocks`), or a + /// scattered read's, at most one vectored read's worth (`Scatter`). iovecs: [][]u8, + /// A landing job's reads that do not scatter (`landJob`), allocated + /// on first use: as large as a job's blocks, page aligned for + /// O_DIRECT. + bounce: []align(std.heap.page_size_min) u8 = &.{}, + bounce_capacity: usize, - fn init(allocator: std.mem.Allocator, maximum_blocks: usize) !Scratch { + fn init(allocator: std.mem.Allocator, maximum_blocks: usize, block_size: usize) !Scratch { return .{ .allocator = allocator, - .iovecs = try allocator.alloc([]u8, maximum_blocks), + .iovecs = try allocator.alloc([]u8, @max(maximum_blocks, max_positional_iovecs)), + .bounce_capacity = maximum_blocks * block_size, }; } + /// The ranges a `Scatter` gathers before it reads. + fn scatterIovecs(self: *const Scratch) [][]u8 { + return self.iovecs[0..max_positional_iovecs]; + } + + fn bounceBuffer(self: *Scratch, len: usize) ![]u8 { + std.debug.assert(len <= self.bounce_capacity); + if (self.bounce.len == 0) self.bounce = try self.allocator.alignedAlloc(u8, .fromByteUnits(std.heap.page_size_min), self.bounce_capacity); + return self.bounce[0..len]; + } + fn deinit(self: *Scratch) void { + self.allocator.free(self.bounce); self.allocator.free(self.iovecs); self.* = undefined; } @@ -1461,6 +1711,7 @@ const ReadRequest = struct { const blocks = loader.credits.slotBlocks(self.slot)[0..job.blockLen(block_size)]; switch (job.kind) { .dma_read => |read| try self.readBlocks(scratch, blocks, read), + .dma_land => |land| try landJob(loader, self.plan, land, blocks, scratch), // Reads nothing: its transfers read the zero block. .dma_fill => return Dma.enqueue(self, loader.blocks[0..1]), } @@ -1552,6 +1803,158 @@ fn readSlot(loader: *Loader, slot: *SourceSlot, iovecs: [][]u8, file_offset: u64 _ = loader.metrics.read_bytes.fetchAdd(@intCast(bytes_read), .monotonic); } +/// Reads a landing job's extents and lands their items' runs in its blocks +/// (`Plan.Item`, `walkItem`). An extent marked for it scatters +/// straight into the blocks, `max_positional_iovecs` ranges per call; any +/// other lands in the worker's bounce buffer and is copied from there: a +/// direct read, whose ranges must be aligned. +fn landJob( + loader: *Loader, + plan: *const Plan, + land: Plan.Job.Land, + blocks: []const host_memory.Block, + scratch: *ReadRequest.Scratch, +) !void { + for (plan.extents[land.extent_start..][0..land.extent_len]) |extent| { + try landExtent(loader, extent, plan.items[extent.item_start..][0..extent.item_len], plan.landings, blocks, scratch); + } +} + +fn landExtent( + loader: *Loader, + extent: Plan.Extent, + items: []const Plan.Item, + landings: []const u64, + blocks: []const host_memory.Block, + scratch: *ReadRequest.Scratch, +) !void { + const block_size = loader.plan_config.block_size; + if (extent.scatter) { + std.debug.assert(extent.len == extent.minimum_len); + var scatter: Scatter = .{ + .loader = loader, + .slot = extent.source_slot, + .iovecs = scratch.scatterIovecs(), + .file_offset = extent.file_offset, + .blocks = blocks, + }; + for (items) |item| try walkItem(item, landings, &scatter); + try scatter.flush(); + std.debug.assert(scatter.file_offset == extent.file_offset + extent.len); + return; + } + const bounce = try scratch.bounceBuffer(extent.len); + var buffers = [_][]u8{bounce}; + try readSlot(loader, extent.source_slot, &buffers, extent.file_offset, extent.minimum_len); + var sink: CopySink = .{ .bounce = bounce, .file_offset = extent.file_offset, .blocks = blocks, .block_size = block_size }; + for (items) |item| try walkItem(item, landings, &sink); +} + +/// Hands `sink` every run of `item` as `place(file offset, landing, len)`, +/// in run order, cut where the destination's box changes +/// (`ShardGrid.runEnd`). +fn walkItem(item: Plan.Item, landings: []const u64, sink: anytype) !void { + const table = landings[item.landing_start..][0..item.destination.grid.box_count]; + try walkRuns(item.lattice, item.first, item.end, item.clip_start, item.clip_end, &item.destination.grid, table, sink); +} + +/// `walkItem` for runs `first` to `end` of `lattice`, clipped to the +/// destination range `[clip_start, clip_end)`. +fn walkRuns(lattice: backend.Lattice, first: u64, end: u64, clip_start: u64, clip_end: u64, grid: *const ShardGrid, table: []const u64, sink: anytype) !void { + if (first >= end) return; + var runs: backend.Lattice.Cursor = .init(lattice, first, end); + while (runs.next()) |run| { + var at = @max(run.dst, clip_start); + const stop = @min(run.dst + lattice.width, clip_end); + var source = run.src + (at - run.dst); + while (at < stop) { + const piece_end = grid.runEnd(at, stop); + const box, const box_offset = grid.locate(at); + try sink.place(source, table[box] +% box_offset, piece_end - at); + source += piece_end - at; + at = piece_end; + } + } +} + +/// The ranges a `Scatter` gathers before it reads, as many as one direct +/// read takes. +const max_positional_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 else std.posix.IOV_MAX; + +/// A read scattered over memory ranges as they come, in file order and +/// back to back from `file_offset` of `slot`'s file: a range that continues +/// the previous one in memory merges with it, and one call reads at most +/// `iovecs.len` ranges. Fed the runs of a walk landing in `blocks` +/// (`place`), or memory ranges directly (`add`). Every byte read must +/// exist: a scattered read has no alignment padding. +const Scatter = struct { + loader: *Loader, + slot: *SourceSlot, + iovecs: [][]u8, + count: usize = 0, + /// Bytes the pending ranges cover. + pending: usize = 0, + /// The file offset of the first byte not read yet. + file_offset: u64, + blocks: []const host_memory.Block = &.{}, + + fn add(self: *Scatter, range: []u8) !void { + if (self.count != 0) { + const last = &self.iovecs[self.count - 1]; + if (last.ptr + last.len == range.ptr) { + last.len += range.len; + self.pending += range.len; + return; + } + } + if (self.count == self.iovecs.len) try self.flush(); + self.iovecs[self.count] = range; + self.count += 1; + self.pending += range.len; + } + + /// A run's landing in the blocks, split at block boundaries. + fn place(self: *Scatter, source: u64, landing: u64, len: u64) !void { + std.debug.assert(source == self.file_offset + self.pending); + var at: usize = @intCast(landing); + var rest: usize = @intCast(len); + while (rest != 0) { + const piece = blockPiece(self.blocks, self.loader.plan_config.block_size, at, rest); + try self.add(piece); + at += piece.len; + rest -= piece.len; + } + } + + fn flush(self: *Scatter) !void { + if (self.count == 0) return; + try readSlot(self.loader, self.slot, self.iovecs[0..self.count], self.file_offset, self.pending); + self.file_offset += self.pending; + self.count = 0; + self.pending = 0; + } +}; + +/// Runs copied from a read in the bounce buffer, which starts at +/// `file_offset`. +const CopySink = struct { + bounce: []const u8, + file_offset: u64, + blocks: []const host_memory.Block, + block_size: usize, + + fn place(self: *CopySink, source: u64, landing: u64, len: u64) !void { + var at: usize = @intCast(landing); + var rest = self.bounce[@intCast(source - self.file_offset)..][0..@intCast(len)]; + while (rest.len != 0) { + const piece = blockPiece(self.blocks, self.block_size, at, rest.len); + @memcpy(piece, rest[0..piece.len]); + rest = rest[piece.len..]; + at += piece.len; + } + } +}; + /// The bytes from offset `at` of `blocks` taken end to end, `block_size` /// bytes each, up to `len` of them or the end of that block. fn blockPiece(blocks: []const host_memory.Block, block_size: usize, at: usize, len: usize) []u8 { @@ -2377,12 +2780,16 @@ const CompositeTest = struct { const Options = struct { block_size: usize, request_size: usize, + alignment: usize = 0, + /// By default nothing is staged. + staging_gain: usize = std.math.maxInt(usize), /// Whether some piece holds a's last byte and b's first one. cross_part_piece: bool, }; - /// The replay reads each read job's blocks from the two files as its - /// worker would (`jobBlocks`). + /// A DMA job is staged where it saves `staging_gain` submissions; the + /// replay reads each job's blocks from the two files as its worker + /// would (`jobBlocks`). fn run(platform: *const Platform, shape: Shape, options: Options) !void { const allocator = std.testing.allocator; const io = std.testing.io; @@ -2440,7 +2847,8 @@ const CompositeTest = struct { plans[plan_count] = try prepareTestPlan(arena.allocator(), &parts, order[start..end], .{ .block_size = block_size, .request_size = options.request_size, - }, 0); + .minimum_staging_gain = options.staging_gain, + }, options.alignment); start = end; } try std.testing.expectEqual(@as(usize, 2), plan_count); @@ -2482,9 +2890,10 @@ const CompositeTest = struct { try std.testing.expectEqual(@as(usize, 0), transfer.landing); break :fill zeros[0..transfer.len]; } else read: { + // a's last byte and b's first one in the same piece, + // where a read's blocks hold the file in order. const file_offset = job.file_offset + transfer.landing; - // a's last byte and b's first one in the same piece. - if (plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; + if (job.kind == .dma_read and plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; break :read blocks[transfer.landing..][0..transfer.len]; }; const index: usize = if (transfer.destination == &destinations[0]) 0 else 1; @@ -2501,16 +2910,33 @@ const CompositeTest = struct { try replays[1].expect(allocator, second_bytes[64..128]); } - /// A read job's blocks taken end to end once its worker has read it, - /// in file order. + /// A DMA job's blocks taken end to end once its worker has read it: + /// the read in file order, or as its worker lands a landing job + /// (`landJob`). fn jobBlocks(allocator: std.mem.Allocator, loader: *Loader, plan: *const Plan, job: Plan.Job) ![]u8 { const block_size = loader.plan_config.block_size; const bytes = try allocator.alloc(u8, job.blockLen(block_size) * block_size); errdefer allocator.free(bytes); @memset(bytes, 0xaa); - const read = job.kind.dma_read; - var buffers = [_][]u8{bytes[0..read.len]}; - try readSlot(loader, plan.source_slot, &buffers, job.file_offset, read.minimum_len); + const land = switch (job.kind) { + .dma_read => |read| { + // The end of the file cuts a widened read's padding. + var buffers = [_][]u8{bytes[0..read.len]}; + try readSlot(loader, plan.source_slot, &buffers, job.file_offset, read.minimum_len); + return bytes; + }, + .dma_land => |land| land, + .dma_fill => unreachable, + }; + // A bounce buffer as large as any of the job's reads. + var longest: usize = land.len; + for (plan.extents[land.extent_start..][0..land.extent_len]) |extent| longest = @max(longest, extent.len); + const blocks = try allocator.alloc(host_memory.Block, job.blockLen(block_size)); + defer allocator.free(blocks); + for (blocks, 0..) |*block, index| block.* = bytes[index * block_size ..][0..block_size]; + var scratch: ReadRequest.Scratch = try .init(allocator, Planner.blockCount(longest, block_size), block_size); + defer scratch.deinit(); + try landJob(loader, plan, land, blocks, &scratch); return bytes; } @@ -2617,6 +3043,12 @@ test "composite destinations place each part and zero-fill the rest of every dev // cut every row apart and every fill piece to one block. try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .cross_part_piece = scenario[2] }); try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 32, .cross_part_piece = false }); + // Staged wherever it saves a submission: the column-sharded jobs + // land grouped by device, in reads exact or widened. + for ([_]usize{ 0, 8 }) |alignment| { + try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .alignment = alignment, .staging_gain = 1, .cross_part_piece = scenario[2] }); + try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 48, .alignment = alignment, .staging_gain = 1, .cross_part_piece = false }); + } } } } @@ -3591,3 +4023,67 @@ test "source slots keep their own copy of each path" { // first string would miss the second lookup. try std.testing.expectEqual(@as(usize, 1), loader.source_slots.count()); } + +test "column-sharded tensors load as one DMA piece per device and block, read exact or widened" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 4); + defer devices.deinit(); + const mesh = try devices.mesh("tp"); + + // Rows of 32 bytes, 8 per device; then 12,000 one-byte pieces, more + // than one vectored read takes. + var rows: [64 * 32]u8 = undefined; + var cols: [1024 * 32]u8 = undefined; + var narrow: [3000 * 4]u8 = undefined; + testing.pattern(&rows, 1); + testing.pattern(&cols, 2); + testing.pattern(&narrow, 3); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "staged.safetensors", ®istry, &.{ + .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &rows }, + .{ .name = "cols", .dtype = "U16", .dims = &.{ 1024, 16 }, .bytes = &cols }, + .{ .name = "narrow", .dtype = "U8", .dims = &.{ 3000, 4 }, .bytes = &narrow }, + }); + const names = [_][]const u8{ "rows", "cols", "narrow" }; + const shapes = [_]Shape{ + Shape.init(.{ .row = 64, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), + Shape.init(.{ .row = 1024, .col = 16 }, .u16).withPartitioning(&mesh, .{ .col = .model }), + Shape.init(.{ .row = 3000, .col = 4 }, .u8).withPartitioning(&mesh, .{ .col = .model }), + }; + const globals = [_][]const u8{ &rows, &cols, &narrow }; + const row_pieces = 1024 * 4 + 3000 * 4; + + for ([_]usize{ 0, 64 }) |alignment| { + var parts: [names.len]LoadPart = undefined; + var specs: [names.len]LoadSpec = undefined; + var outputs: [names.len]Buffer = undefined; + for (&parts, &specs, &outputs, names, shapes) |*part, *spec, *output, name, shape| { + part.* = .whole(registry.tensors.getPtr(name).?); + spec.* = .{ .shape = shape, .output = output, .parts = part[0..1] }; + } + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + loader.reads.alignment = alignment; + const batch = try loader.submit(&specs, .none); + // Every plan is published once `submit` returns. + var staged_jobs: usize = 0; + var submissions: usize = 0; + for (batch.plans.items) |plan| { + for (plan.jobs) |job| staged_jobs += @intFromBool(job.kind == .dma_land); + submissions += plan.events.len; + } + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + for (&outputs, globals) |*output, global| try testing.expectShards(allocator, io, output, global, .default); + try std.testing.expect(staged_jobs >= 1); + try std.testing.expect(submissions < row_pieces / 100); + // Every credit is free once the workers stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + } +} From 288c4375ed8c8543b8033dca717c056036b21f9c Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 15:49:21 +0100 Subject: [PATCH 084/103] zml/io: gather strided parts in the direct backend A source whose strided parts do not expand into contiguous ones (a row interleave, a transpose, a broadcast, a block-scale swizzle) was refused; it now goes to the backend as strided parts, and the direct backend plans its destination whole. The destination's logical bytes are cut into ranges of at most a request, less two units of the widest alignment a file may take. Each range is one landing job: for every part with runs in it, the job reads the source range of those runs, or the part's whole source range when its runs are out of order, from whichever file holds it, copies the runs into its blocks box by box, zeroes the bytes no part covers, and sends each box as one piece per block. The copy loops over whole rows of the two innermost dimensions, and moves runs of 1, 2, 4, 8 or 16 bytes as one word. Since such a job may read a part's whole source range, the front end asks the backend what each strided part reads and refuses at start, as InvalidBinding logged with both numbers, a part that needs more than one read: the request of the smallest block calibration may pick, so a refusal does not depend on the pick. It also refuses strided parts into host memory. The buffered backend reads any part. The tests load row interleaves across files, broadcasts, transposes and a block-scale swizzle over one, two and four devices, with reads exact and widened, check the read bounds of local and remote parts, and fuzz copyRuns at word widths. Co-Authored-By: Claude Opus 5.5 --- zml/io/backend.zig | 165 ++++++++++++-- zml/io/direct_loader.zig | 470 +++++++++++++++++++++++++++++++++++---- zml/io/loader.zig | 326 ++++++++++++++++++++++++--- 3 files changed, 865 insertions(+), 96 deletions(-) diff --git a/zml/io/backend.zig b/zml/io/backend.zig index 44dfdbfb..2cc96fd4 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -64,7 +64,8 @@ pub const Options = struct { }; /// One destination, its caller-owned output and the source ranges that fill -/// it, as the front end checked them (`checkParts`): backends trust them. +/// it, as the front end checked them (`checkParts`, `Backend.stridedRead`): +/// backends trust them. /// Everything a spec names (its parts, their dimensions, the sources, the /// output) stays alive until its await; backends borrow it. pub const LoadSpec = struct { @@ -161,17 +162,75 @@ pub const Lattice = struct { /// destination side: runs land in index order without overlapping. /// Needs `spans` not null. pub fn dstOrdered(self: Lattice) bool { - var extent = self.width; - var d = self.dims.len; + return ordered(self.dims, self.width, .dst, false); + } + + /// Whether the source runs come in index order without overlapping, + /// gaps allowed: the runs of an index range lie between the first's + /// start and the last's end. Needs `spans` not null. + pub fn srcOrdered(self: Lattice) bool { + return ordered(self.dims, self.width, .src, false); + } + + /// Whether the source runs are back to back in index order: an index + /// range reads one contiguous source range, each byte once. Needs + /// `spans` not null. + pub fn srcTiled(self: Lattice) bool { + return ordered(self.dims, self.width, .src, true); + } + + /// Whether each dimension steps past everything inside it on `side`, + /// exactly with `exact`. The extents it sums are bounded by `spans`. + fn ordered(dims: []const Dim, width: u64, comptime side: enum { src, dst }, exact: bool) bool { + var extent = width; + var d = dims.len; while (d > 0) { d -= 1; - if (self.dims[d].count < 2) continue; - if (self.dims[d].dst_stride < extent) return false; - extent += (self.dims[d].count - 1) * self.dims[d].dst_stride; + if (dims[d].count < 2) continue; + const stride = if (side == .src) dims[d].src_stride else dims[d].dst_stride; + if (if (exact) stride != extent else stride < extent) return false; + extent += (dims[d].count - 1) * stride; } return true; } + /// The first run, in index order, that ends after destination offset + /// `offset`; `runs()` when none does. Needs `dstOrdered`. + pub fn firstEndingAfter(self: Lattice, offset: u64) u64 { + var low: u64 = 0; + var high = self.runs(); + while (low < high) { + const middle = low + (high - low) / 2; + if (self.at(middle).dst + self.width > offset) high = middle else low = middle + 1; + } + return low; + } + + /// The first run that starts at destination offset `offset` or later; + /// `runs()` when none does. Needs `dstOrdered`. + pub fn firstStartingFrom(self: Lattice, offset: u64) u64 { + var low: u64 = 0; + var high = self.runs(); + while (low < high) { + const middle = low + (high - low) / 2; + if (self.at(middle).dst >= offset) high = middle else low = middle + 1; + } + return low; + } + + /// The destination bytes of the runs inside `[start, end)`. + pub fn coveredIn(self: Lattice, start: u64, end: u64) u64 { + const first = self.firstEndingAfter(start); + const last = self.firstStartingFrom(end); + if (first >= last) return 0; + var covered = (last - first) * self.width; + const head = self.at(first).dst; + if (head < start) covered -= start - head; + const tail = self.at(last - 1).dst + self.width; + if (tail > end) covered -= tail - end; + return covered; + } + /// Runs `first` to `end` (exclusive), in index order, without a /// division per run. pub const Cursor = struct { @@ -218,13 +277,72 @@ pub const Lattice = struct { /// Copies runs `first` to `end` (exclusive): the `width` bytes at /// `src + run.src` to `dst + run.dst`, `src` and `dst` being the - /// addresses the offsets count from (wrapping). + /// addresses the offsets count from (wrapping). Whole rows of the two + /// innermost dimensions go through nested loops, and a run of 1, 2, 4, + /// 8 or 16 bytes is one load and one store: a swizzle's 4-byte runs + /// cost a few cycles each instead of a cursor step and a `memcpy`. pub fn copyRuns(self: Lattice, first: u64, end: u64, dst: usize, src: usize) void { + switch (self.width) { + inline 1, 2, 4, 8, 16 => |width| self.copyRunsOf(width, first, end, dst, src), + else => self.copyRunsOf(0, first, end, dst, src), + } + } + + /// `copyRuns` for runs of `width` bytes, or of `self.width` when zero. + fn copyRunsOf(self: Lattice, comptime width: usize, first: u64, end: u64, dst: usize, src: usize) void { + if (first >= end) return; + const inner = self.dims[self.dims.len - @min(self.dims.len, 2) ..]; + var row: u64 = 1; + for (inner) |d| row *= d.count; + const rows_start = std.math.divCeil(u64, first, row) catch unreachable; + const rows_end = end / row; + if (rows_start >= rows_end) return self.copyEach(width, first, end, dst, src); + self.copyEach(width, first, rows_start * row, dst, src); + const outer: Lattice = .{ .src_offset = self.src_offset, .dst_offset = self.dst_offset, .width = self.width, .dims = self.dims[0 .. self.dims.len - inner.len] }; + var rows: Cursor = .init(outer, rows_start, rows_end); + while (rows.next()) |start| { + const row_dst = dst +% start.dst; + const row_src = src +% start.src; + switch (inner.len) { + 0 => copyRun(width, self.width, row_dst, row_src), + 1 => copyLine(width, self.width, inner[0], row_dst, row_src), + else => { + var line_dst = row_dst; + var line_src = row_src; + for (0..inner[0].count) |_| { + copyLine(width, self.width, inner[1], line_dst, line_src); + line_dst +%= inner[0].dst_stride; + line_src +%= inner[0].src_stride; + } + }, + } + } + self.copyEach(width, rows_end * row, end, dst, src); + } + + fn copyEach(self: Lattice, comptime width: usize, first: u64, end: u64, dst: usize, src: usize) void { if (first >= end) return; - const width: usize = @intCast(self.width); var cursor: Cursor = .init(self, first, end); - while (cursor.next()) |run| { - @memcpy(@as([*]u8, @ptrFromInt(dst +% run.dst))[0..width], @as([*]const u8, @ptrFromInt(src +% run.src))[0..width]); + while (cursor.next()) |run| copyRun(width, self.width, dst +% run.dst, src +% run.src); + } + + inline fn copyLine(comptime width: usize, len: u64, d: Dim, dst: usize, src: usize) void { + var to = dst; + var from = src; + for (0..d.count) |_| { + copyRun(width, len, to, from); + to +%= d.dst_stride; + from +%= d.src_stride; + } + } + + inline fn copyRun(comptime width: usize, len: u64, dst: usize, src: usize) void { + if (width == 0) { + const n: usize = @intCast(len); + @memcpy(@as([*]u8, @ptrFromInt(dst))[0..n], @as([*]const u8, @ptrFromInt(src))[0..n]); + } else { + const Word = @Int(.unsigned, 8 * width); + @as(*align(1) Word, @ptrFromInt(dst)).* = @as(*align(1) const Word, @ptrFromInt(src)).*; } } }; @@ -310,6 +428,25 @@ pub const Backend = union(enum) { }; } + /// `read`, the source bytes every job of `part` reads whole, widened to + /// its file's alignment (0 when its jobs read only their own range: + /// source runs back to back, a contiguous part included), and `limit`, + /// the most one read takes: the front end refuses a part whose `read` + /// passes `limit`. Needs `checkParts`. The direct backend's limit is its + /// smallest read, whichever block calibration picks: the checkpoint's + /// read chunk (8 MiB locally) rounded up to whole blocks of the smallest + /// it may pick, 2 MiB or a pinned `dma_block_size` + /// (`direct_loader.Loader.stridedRead`); the buffered backend reads any + /// part. + pub fn stridedRead(self: Backend, part: LoadPart) StridedRead { + return switch (self) { + .direct => |loader| loader.stridedRead(part), + .buffered => .{ .read = 0, .limit = std.math.maxInt(u64) }, + }; + } + + pub const StridedRead = struct { read: u64, limit: u64 }; + /// Waits for and retires `submission`, which this backend published. /// Its batch is dangling afterwards. pub fn await(self: Backend, submission: Submission) !void { @@ -366,10 +503,12 @@ test "the cursor walks a range's runs where at places them, and copyRuns copies var actual: [4096]u8 = undefined; var dims_buffer: [5]Lattice.Dim = undefined; for (0..2000) |_| { - const lattice = randomLattice(random, dims_buffer[0..random.intRangeAtMost(usize, 0, dims_buffer.len)]); + var lattice = randomLattice(random, dims_buffer[0..random.intRangeAtMost(usize, 0, dims_buffer.len)]); + // The widths `copyRuns` moves as one word, and others; a wider run + // may overlap its neighbours, which no part does. + if (random.boolean()) lattice.width = ([_]u64{ 1, 2, 4, 8, 16 })[random.uintLessThan(usize, 5)]; const spans = lattice.spans().?; - if (lattice.src_offset + spans.src > source.len or lattice.dst_offset + spans.dst > expected.len) continue; - try std.testing.expect(lattice.dstOrdered()); + if (lattice.src_offset + spans.src > source.len or lattice.dst_offset + spans.dst > expected.len or !lattice.dstOrdered()) continue; const runs = lattice.runs(); const first = random.uintAtMost(u64, runs); const end = random.intRangeAtMost(u64, first, runs); diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 8b8239b0..9b0da89f 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -49,7 +49,15 @@ //! its first part, as a last fill job that reads nothing and transfers from //! the zero block, which no credit owns. The job of a tensor sharded along //! any axis but its first is staged by destination where that saves -//! submissions (`JobCut.placeDma`). +//! submissions (`JobCut.placeDma`). Strided parts are gathered +//! (`JobCut.appendGatherJobs`), their runs copied out of the worker's bounce +//! buffer by `Lattice.copyRuns`. Block-scale swizzles gather this way, as +//! runs of four bytes. Runs under 4 KiB cost host copies, where a program +//! costs a compile and staging memory: on DeepSeek-V4.1 over two GB300, +//! laying the MXFP4 experts out here (gate and up rows interleaved, scales +//! swizzled) copied 77 to 189 GiB through bounce buffers, 17 to 36 s of +//! worker time; as programs it copied 1 to 3 GiB and loaded 0.1 to 1.1 s +//! faster, for 4 to 6 GiB more device memory at the peak. //! //! Locks. Where several pumps are held, they are taken in device order. A //! pump's mutex may be held while a released request gives its credit back @@ -136,7 +144,8 @@ pub const Loader = struct { created_at: std.Io.Timestamp, /// Submissions so far; the next batch's sequence number. batch_count: usize = 0, - /// The block size and the request size, for the whole load. + /// The block size, the request size and the gather limit, for the whole + /// load. plan_config: Planner.Config, /// How each file is read: exact, or aligned for direct I/O. reads: Planner.Reads, @@ -227,6 +236,7 @@ pub const Loader = struct { .plan_config = .{ .block_size = block_size, .request_size = request_size, + .gather_limit = backend.requestSize(opts.read_hints.read_chunk_size, std.mem.min(usize, opts.dmaBlockSizes()), reads.alignment).?, }, .reads = reads, }; @@ -257,11 +267,12 @@ pub const Loader = struct { /// as soon as it exists behind every earlier submission, then seals the /// batch: work on the first file starts while the rest is planned. The /// batch owns its destinations and parts until `awaitBatch` retires it. - /// The specs keep the `LoadSpec` contract, which the front end checked, - /// and this backend's: device memory and contiguous parts. A planning - /// failure fails the loader (a partial submission can never complete): - /// the batch is awaited here and the caller sees only the first error. - /// A failed loader refuses every submit with its first error. + /// The specs keep the `LoadSpec` contract and `stridedRead`'s limit, + /// which the front end checked, and this backend's: device memory. A + /// planning failure fails the loader (a partial submission can never + /// complete): the batch is awaited here and the caller sees only the + /// first error. A failed loader refuses every submit with its first + /// error. pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { try self.checkOpen(); const batch = batch: { @@ -289,6 +300,12 @@ pub const Loader = struct { return batch; } + /// The source bytes every job of `part` reads whole, against one read + /// (`Planner.stridedRead`). + pub fn stridedRead(self: *const Loader, part: LoadPart) backend.Backend.StridedRead { + return Planner.stridedRead(self.plan_config, self.reads, part); + } + /// Waits for the batch's last completion unit, retires it and returns /// the loader's sticky error if the loader failed. Destroying the batch /// destroys its transfer managers, and PJRT's manager destructor errors @@ -386,8 +403,8 @@ pub const Loader = struct { const destination_parts = parts[next_part..][0..spec.parts.len]; next_part += spec.parts.len; var covered_bytes: u64 = 0; + var gathered = false; for (spec.parts, destination_parts) |part, *slot| { - std.debug.assert(part.lattice.dims.len == 0); // submit slot.* = .{ .source = part.source, .source_slot = try self.sourceSlot(part.source.file_uri), @@ -395,13 +412,18 @@ pub const Loader = struct { .lattice = part.lattice, }; covered_bytes += part.lattice.width; + gathered = gathered or part.lattice.dims.len != 0; + const strided = self.stridedRead(part); + std.debug.assert(strided.read <= strided.limit); // backend.Backend.stridedRead } destination.* = .{ .shape = spec.shape, .grid = grid, .output = spec.output, .parts = destination_parts, - .unread_bytes = .init(covered_bytes), + .gathered = gathered, + // A gathered destination's jobs land every byte, zeros too. + .unread_bytes = .init(if (gathered) grid.total else covered_bytes), .progress = progress, }; } @@ -515,13 +537,18 @@ pub const Loader = struct { var transfers: usize = 0; var dma_submissions: usize = 0; var kinds: std.EnumArray(std.meta.Tag(Plan.Job.Kind), usize) = .initFill(0); + var gather_jobs: usize = 0; for (batch.plans.items) |plan| { planning_ns += plan.planning_ns; source_bytes += plan.source_bytes; jobs += plan.jobs.len; transfers += plan.transfers.len; dma_submissions += plan.events.len; - for (plan.jobs) |job| kinds.getPtr(job.kind).* += 1; + for (plan.jobs) |job| { + kinds.getPtr(job.kind).* += 1; + // A gathered destination's landing jobs land it alone. + if (job.kind == .dma_land and plan.transfersOf(job)[0].destination.gathered) gather_jobs += 1; + } } const fill_jobs = kinds.get(.dma_fill); const source_jobs = jobs - fill_jobs; @@ -531,7 +558,7 @@ pub const Loader = struct { else @as(f64, @floatFromInt(batch.parts.len)) / @as(f64, @floatFromInt(source_jobs)); - load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, staged_jobs={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ + load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, staged_jobs={d}, gather_jobs={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ batch.sequence, batch.plans.items.len, @as(f64, @floatFromInt(planning_ns)) / std.time.ns_per_s, @@ -541,7 +568,8 @@ pub const Loader = struct { batch.parts.len, transfers, dma_submissions, - kinds.get(.dma_land), + kinds.get(.dma_land) - gather_jobs, + gather_jobs, coalescing_ratio, average_read_size, }); @@ -664,12 +692,26 @@ const Destination = struct { /// covers are written from the loader's zero block, by the fill job of /// the plan holding `parts[0]`. parts: []Part, - /// Bytes not read yet, the parts' widths at first. The read that - /// brings it to zero completes the destination's item on `progress`. + /// Fed by strided parts: planned whole, by logical ranges, with the + /// plan holding `parts[0]` (`Planner.JobCut.appendGatherJobs`), whose + /// jobs also write its zero fill. + gathered: bool = false, + /// Bytes not read yet: the parts' widths, or for a gathered destination + /// its whole placed size (its gather jobs land the zeros too). The read + /// that brings it to zero completes the destination's item on + /// `progress`. unread_bytes: std.atomic.Value(u64), progress: std.Progress.Node = .none, state: LazyOnce(DestinationState) = .{}, + /// How its parts are planned, a class at a time in this order + /// (`Planner.sortedPartOrder`): DMA jobs, or gathered whole. + const Delivery = enum { dma, gathered }; + + fn delivery(self: *const Destination) Delivery { + return if (self.gathered) .gathered else .dma; + } + fn ensureState(self: *Destination, allocator: std.mem.Allocator, io: std.Io, platform: *const Platform) StateInitError!*DestinationState { return self.state.ensure(io, DestinationState.init, .{ allocator, platform, self }); } @@ -737,8 +779,10 @@ const Plan = struct { /// A claimable job: its slice of the plan's transfers and what its /// worker does for them. Its request slot is its index. const Job = struct { - /// Where the job's read starts; zero for a fill job, which reads - /// nothing and comes after the plan's reads. + /// Where the job reads first: its read's start, a landing job's + /// first extent's; zero for a job that reads nothing. + /// `Planner.preparePlan` sorts the other jobs by it and appends + /// the fill job after them. file_offset: u64, transfer_start: usize, transfer_len: usize, @@ -751,7 +795,8 @@ const Plan = struct { dma_read: Read, /// Reads its extents, whose items land their bytes in its /// blocks: a read of its plan's file grouped by destination - /// box (`Planner.JobCut.stage`). + /// box (`Planner.JobCut.stage`), or a gathered destination's + /// range (`Planner.JobCut.appendGatherJobs`). dma_land: Land, /// Reads nothing: every transfer goes from the loader's zero /// block (`Planner.JobCut.appendFillTransfers`). @@ -768,11 +813,14 @@ const Plan = struct { /// A landing job: `len` bytes in its blocks, landed by the /// `extent_len` extents of the plan's `extents` from - /// `extent_start`, in order. + /// `extent_start`, in order. A gathered range no part covers + /// has none. const Land = struct { len: usize, extent_start: usize, extent_len: usize, + /// Its blocks start zeroed: bytes no part covers. + zeroed: bool = false, }; /// The blocks of its credit's slot the job reads into or lands @@ -876,6 +924,10 @@ const Planner = struct { /// The longest read, a whole number of blocks /// (`backend.requestSize`). request_size: usize, + /// The one-read limit of a strided part (`stridedRead`, which caps + /// it at `request_size`): the request of the smallest block + /// calibration may pick, so a refusal does not follow its pick. + gather_limit: usize = std.math.maxInt(usize), /// Submissions a DMA job must save to be staged (`JobCut.stage`): /// one device's in-flight pieces. minimum_staging_gain: usize = max_dma_pieces_per_device, @@ -885,7 +937,8 @@ const Planner = struct { /// opens it reads it directly under `policy`, decided once per file and /// kept in its slot. A remote file is read exact without being opened /// here. Without a VFS (`io` is none, or no alignment) every file takes - /// `alignment`. + /// `alignment`. A gather job reads files other than its plan's, so it + /// asks here for each. /// Widening buffered reads is not free: a widened read served from the /// page cache costs the DMA sources their block alignment for nothing, /// and warm replicated Llama on four GB300 lost 3-6% when its first @@ -921,6 +974,13 @@ const Planner = struct { slot.alignment = alignment; return alignment; } + + /// The widest alignment `alignmentOf` can give the file, without + /// opening it: 0 for a remote file. + fn boundOf(self: Reads, uri: []const u8) usize { + const vfs = self.vfs orelse return self.alignment; + return if (vfs.isLocal(uri)) self.alignment else 0; + } }; /// Plans `parts` one file at a time and publishes each plan as soon as @@ -952,11 +1012,13 @@ const Planner = struct { } } - /// Plans one file: `order` indexes its `parts` by offset and size - /// (`sortedPartOrder`), `JobCut.cutRuns` cuts them at the file's - /// alignment (`Reads.alignmentOf`), then `JobCut.appendFillJob` appends - /// the zero fill. The plan goes in `arena` (`JobCut.finish`), the cut in - /// `allocator`. + /// Plans one file: `order` indexes its `parts` by delivery class, then + /// offset and size (`sortedPartOrder`). `JobCut.cutRuns` cuts the DMA + /// parts at the file's alignment (`Reads.alignmentOf`), + /// `JobCut.appendGatherJobs` each gathered destination with the file of + /// its first part; the jobs merge in file order, then + /// `JobCut.appendFillJob` appends the zero fill. The plan goes in + /// `arena` (`JobCut.finish`), the cut in `allocator`. fn preparePlan( allocator: std.mem.Allocator, arena: std.mem.Allocator, @@ -967,28 +1029,54 @@ const Planner = struct { ) !*Plan { const source_slot = parts[order[0]].source_slot; for (order) |part_index| std.debug.assert(std.mem.eql(u8, parts[part_index].source.file_uri, parts[order[0]].source.file_uri)); + const dma_end = deliveryEnd(parts, order, 0, .dma); const alignment = try reads.alignmentOf(source_slot); var cut: JobCut = .{ .allocator = allocator, .parts = parts, .source_slot = source_slot, .config = config, + .reads = reads, .alignment = alignment, .maximum_job_len = maximumJobLen(config.request_size, alignment), + // A gather job's extents may be in files read directly while + // this one is not, so its ranges leave room for the widest + // alignment. + .gather_len = maximumJobLen(config.request_size, reads.alignment), }; defer cut.deinit(); - // Jobs are claimed in planning order, which is file order. A - // byte-fair order across destination devices was measured against - // it on the two hosts where the DMA pump is not the ceiling and made - // no difference. - try cut.cutRuns(order); + try cut.cutRuns(order[0..dma_end]); + for (order[dma_end..]) |part_index| { + const part = &parts[part_index]; + std.debug.assert(part.destination.gathered); + if (part == &part.destination.parts[0]) try cut.appendGatherJobs(part.destination); + } + // Jobs are claimed in planning order, which is file order (a + // gather job by its first read). A byte-fair order across + // destination devices was measured against it on the two hosts + // where the DMA pump is not the ceiling and made no difference. + // Each class is in file order already; the stable sort interleaves + // them. + std.mem.sort(Plan.Job, cut.jobs.items, {}, struct { + fn lessThan(_: void, lhs: Plan.Job, rhs: Plan.Job) bool { + return lhs.file_offset < rhs.file_offset; + } + }.lessThan); + // The zero fill of each destination goes out once, with the plan // of its first part; it depends on no read, so one job after this // file's reads carries all of it. - try cut.appendFillJob(order); + try cut.appendFillJob(order[0..dma_end]); return cut.finish(arena); } + /// The end of the parts of `delivery` in `order` from `start` on. + fn deliveryEnd(parts: []const Part, order: []const usize, start: usize, delivery: Destination.Delivery) usize { + var end = start; + while (end < order.len and parts[order[end]].destination.delivery() == delivery) : (end += 1) {} + return end; + } + /// The jobs and transfers of one plan as `preparePlan` cuts them. const JobCut = struct { /// One part's bytes in a read job: the logical range `[start, end)` @@ -1005,9 +1093,12 @@ const Planner = struct { /// The file every job of the plan reads. source_slot: *SourceSlot, config: Config, + reads: Reads, /// The alignment of the plan's file (`Reads.alignmentOf`). alignment: usize, maximum_job_len: usize, + /// The longest logical range of a gather job. + gather_len: usize, jobs: std.ArrayList(Plan.Job) = .empty, transfers: std.ArrayList(Plan.Transfer) = .empty, items: std.ArrayList(Plan.Item) = .empty, @@ -1058,15 +1149,15 @@ const Planner = struct { self.jobs.deinit(allocator); } - /// Cuts the runs of `order` (part indices in file order) into jobs: - /// each run of touching or overlapping ranges into the minimum number - /// of jobs, cut at part-safe boundaries where that number allows. No - /// job depends on another, since every DMA piece is submitted as soon - /// as its block is read. With an alignment (`Reads.alignmentOf`), a - /// job's read widens to aligned bounds around its range - /// (`maximumJobLen` leaves room for it), and the transfers address - /// the widened read. Each job's segments are placed in its blocks - /// by `placeDma`. + /// Cuts the runs of `order` (the DMA parts' indices in file order) + /// into jobs: each run of touching or overlapping ranges into the + /// minimum number of jobs, cut at part-safe boundaries where that + /// number allows. No job depends on another, since every DMA piece + /// is submitted as soon as its block is read. With an alignment + /// (`Reads.alignmentOf`), a job's read widens to aligned bounds + /// around its range (`maximumJobLen` leaves room for it), and the + /// transfers address the widened read. Each job's segments are + /// placed in its blocks by `placeDma`. fn cutRuns(self: *JobCut, order: []const usize) !void { const allocator = self.allocator; const parts = self.parts; @@ -1347,6 +1438,133 @@ const Planner = struct { } } + /// Plans a gathered destination (`Destination.gathered`) whole: its + /// logical bytes cut into ranges of at most `gather_len`, each one + /// job. For every part with runs in the range the job reads the + /// source range those runs come from (an `Extent`, from whichever + /// file holds it), and it lands the range in its blocks box by box, + /// one transfer per box and block, its bytes no part covers zero. + /// A part whose source runs are out of order (a broadcast repeats + /// them, a transpose permutes them) reads its whole source range + /// in every job (`stridedRead`). An interleave of members reads + /// each member's rows of the range, so the range lands in one piece + /// per box and block whatever the interleave's granularity. + fn appendGatherJobs(self: *JobCut, destination: *Destination) !void { + const allocator = self.allocator; + const grid = &destination.grid; + const parts = destination.parts; + // reach[i]: the furthest destination byte parts[0..=i] reach, so + // a range's first candidate is found by bisection. + const reach = try allocator.alloc(u64, parts.len); + defer allocator.free(reach); + var furthest: u64 = 0; + for (parts, reach) |*part, *at| { + const spans = part.lattice.spans().?; + furthest = @max(furthest, part.lattice.dst_offset + spans.dst); + at.* = furthest; + } + var start: u64 = 0; + while (start < grid.total) { + const end = @min(grid.total, start + self.gather_len); + const transfer_start = self.transfers.items.len; + const extent_start = self.extents.items.len; + const landing_start = self.landings.items.len; + try self.landings.appendNTimes(allocator, 0, grid.box_count); + var landing: u64 = 0; + for (0..grid.box_count) |box| { + const low = grid.rankIn(box, start); + const high = grid.rankIn(box, end); + if (high == low) continue; + self.landings.items[landing_start + box] = landing -% low; + try self.appendBlocked(transfer_start, .{ + .destination = destination, + .landing = @intCast(landing), + .writer_mask = grid.writers[box], + .destination_offset = @intCast(low), + .len = @intCast(high - low), + }); + landing += high - low; + } + std.debug.assert(landing == end - start); + + var covered: u64 = 0; + var index = std.sort.partitionPoint(u64, reach, start, struct { + fn below(limit: u64, at: u64) bool { + return at <= limit; + } + }.below); + while (index < parts.len and parts[index].lattice.dst_offset < end) : (index += 1) { + const part = &parts[index]; + const first = part.lattice.firstEndingAfter(start); + const last = part.lattice.firstStartingFrom(end); + if (first >= last) continue; + covered += part.lattice.coveredIn(start, end); + try self.appendExtent(part, first, last, start, end, landing_start); + } + std.debug.assert(covered <= end - start); + const job_len: usize = @intCast(end - start); + const extent_len = self.extents.items.len - extent_start; + try self.jobs.append(allocator, .{ + .file_offset = if (extent_len != 0) self.extents.items[extent_start].file_offset else 0, + .transfer_start = transfer_start, + .transfer_len = self.transfers.items.len - transfer_start, + .kind = .{ .dma_land = .{ + .len = job_len, + .extent_start = extent_start, + .extent_len = extent_len, + .zeroed = covered < end - start, + } }, + }); + start = end; + } + } + + /// The extent reading `part`'s runs `first` to `last` (exclusive) + /// clipped to `[start, end)`, with the one item that places them: + /// from the first run's first byte in the range to the last run's + /// last one when the part's source runs are in order, else the + /// part's whole source range. + fn appendExtent(self: *JobCut, part: *const Part, first: u64, last: u64, start: u64, end: u64, landing_start: usize) !void { + const lattice = part.lattice; + const width = lattice.width; + var source_start = lattice.src_offset; + var source_end = lattice.src_offset + lattice.spans().?.src; + const ordered = lattice.srcOrdered(); + if (ordered) { + const head = lattice.at(first); + const tail = lattice.at(last - 1); + source_start = head.src + (start -| head.dst); + source_end = tail.src + width - ((tail.dst + width) -| end); + } + const file_start = part.source.offset + source_start; + const file_end = part.source.offset + source_end; + const alignment = try self.reads.alignmentOf(part.source_slot); + const read_start = if (alignment == 0) file_start else std.mem.alignBackward(u64, file_start, alignment); + const read_end = if (alignment == 0) file_end else std.mem.alignForward(u64, file_end, alignment); + // The range of a part whose runs are back to back is the job's; + // any other part's whole range fits (`stridedRead`). + std.debug.assert(read_end - read_start <= self.config.request_size); + try self.extents.append(self.allocator, .{ + .source_slot = part.source_slot, + .file_offset = read_start, + .len = @intCast(read_end - read_start), + .minimum_len = @intCast(file_end - read_start), + .scatter = alignment == 0 and ordered and lattice.srcTiled(), + .item_start = self.items.items.len, + .item_len = 1, + }); + try self.items.append(self.allocator, .{ + .destination = part.destination, + .lattice = .{ .src_offset = part.source.offset + lattice.src_offset, .dst_offset = lattice.dst_offset, .width = width, .dims = lattice.dims }, + .first = first, + .end = last, + .clip_start = start, + .clip_end = end, + .landing_start = landing_start, + }); + self.source_bytes +|= file_end - file_start; + } + /// Appends `transfer`, cut at block boundaries, merged with the /// previous transfer from `transfer_start` on where block, writers /// and both offsets continue it. @@ -1389,8 +1607,9 @@ const Planner = struct { } /// The zero fill of the destinations whose first part is among - /// `order`, the file's parts, as one job that reads one block, the - /// zero block. + /// `order`, the file's DMA parts, as one job that reads one block, + /// the zero block. A gathered destination's jobs write its zeros + /// themselves. fn appendFillJob(self: *JobCut, order: []const usize) !void { const fill_start = self.transfers.items.len; for (order) |part_index| { @@ -1466,6 +1685,28 @@ const Planner = struct { return len / block_size + @intFromBool(len % block_size != 0); } + /// The source bytes every job of `part` reads whole, against one read + /// (`Config.gather_limit`, at most a request). A part whose source runs + /// are back to back reads only its job's range: 0. Any other counts its + /// whole source range, widened to the widest alignment its file may be + /// read at (`Reads.boundOf`): out of order (a broadcast repeats its + /// runs, a transpose permutes them), every job reads that range + /// (`JobCut.appendExtent`); in order with gaps (a slice along an inner + /// axis), a job reads from its first run to its last, which the gaps + /// stretch past any job's length. + fn stridedRead(config: Config, reads: Reads, part: LoadPart) backend.Backend.StridedRead { + const lattice = part.lattice; + const limit = @min(config.request_size, config.gather_limit); + if (lattice.srcTiled()) return .{ .read = 0, .limit = limit }; + const alignment = @max(reads.boundOf(part.source.file_uri), 1); + const start = part.source.offset + lattice.src_offset; + const end = start + lattice.spans().?.src; + return .{ + .read = std.mem.alignForward(u64, end, alignment) - std.mem.alignBackward(u64, start, alignment), + .limit = limit, + }; + } + /// The longest tensor range one job may cover: the request size, less /// the two alignment units its widened read can add, so a widened job /// still fits a request (a credit's slot). `backend.requestSize` refuses @@ -1477,9 +1718,10 @@ const Planner = struct { return request_size - 2 * alignment; } - /// Part indices sorted by file URI, offset, size and index: the - /// planner's input, taken one file group at a time (`fileGroupEnd`), in - /// file order. + /// Part indices sorted by file URI, delivery class + /// (`Destination.Delivery`), offset, size and index: the planner's + /// input, taken one file group at a time (`fileGroupEnd`), each class + /// in file order. fn sortedPartOrder(allocator: std.mem.Allocator, parts: []const Part) ![]usize { const order = try allocator.alloc(usize, parts.len); for (order, 0..) |*index, i| index.* = i; @@ -1491,6 +1733,9 @@ const Planner = struct { const right = ctx.parts[rhs]; const uri_order = std.mem.order(u8, left.source.file_uri, right.source.file_uri); if (uri_order != .eq) return uri_order == .lt; + const left_delivery = left.destination.delivery(); + const right_delivery = right.destination.delivery(); + if (left_delivery != right_delivery) return @backingInt(left_delivery) < @backingInt(right_delivery); const left_offset = left.fileRange().start; const right_offset = right.fileRange().start; if (left_offset != right_offset) return left_offset < right_offset; @@ -1781,7 +2026,8 @@ fn readFilePositionalAllV( } /// Every worker read goes through here, under a read permit: a failure is -/// recorded with the file. Once the loader failed, returns its first error. +/// recorded with the file, since a landing job's extents can come from files +/// other than its plan's. Once the loader failed, returns its first error. /// Consumes `iovecs` (`readFilePositionalAllV`). fn readSlot(loader: *Loader, slot: *SourceSlot, iovecs: [][]u8, file_offset: u64, minimum_len: usize) !void { const io = loader.io; @@ -1807,7 +2053,8 @@ fn readSlot(loader: *Loader, slot: *SourceSlot, iovecs: [][]u8, file_offset: u64 /// (`Plan.Item`, `walkItem`). An extent marked for it scatters /// straight into the blocks, `max_positional_iovecs` ranges per call; any /// other lands in the worker's bounce buffer and is copied from there: a -/// direct read, whose ranges must be aligned. +/// direct read, whose ranges must be aligned, and a gather read whose +/// source runs are out of order or repeat (a transpose, a broadcast). fn landJob( loader: *Loader, plan: *const Plan, @@ -1815,6 +2062,9 @@ fn landJob( blocks: []const host_memory.Block, scratch: *ReadRequest.Scratch, ) !void { + if (land.zeroed) { + for (blocks) |block| @memset(block[0..loader.plan_config.block_size], 0); + } for (plan.extents[land.extent_start..][0..land.extent_len]) |extent| { try landExtent(loader, extent, plan.items[extent.item_start..][0..extent.item_len], plan.landings, blocks, scratch); } @@ -1847,7 +2097,7 @@ fn landExtent( var buffers = [_][]u8{bounce}; try readSlot(loader, extent.source_slot, &buffers, extent.file_offset, extent.minimum_len); var sink: CopySink = .{ .bounce = bounce, .file_offset = extent.file_offset, .blocks = blocks, .block_size = block_size }; - for (items) |item| try walkItem(item, landings, &sink); + for (items) |item| try copyItem(item, landings, &sink); } /// Hands `sink` every run of `item` as `place(file offset, landing, len)`, @@ -1877,6 +2127,47 @@ fn walkRuns(lattice: backend.Lattice, first: u64, end: u64, clip_start: u64, cli } } +/// `walkItem` from the bounce buffer, a stretch at a time where that pays: +/// a stretch of the destination landing in one piece, inside one run of +/// the grid and one block, has its whole runs copied by +/// `Lattice.copyRuns`, and only the runs its ends cut go one at a time. A +/// stretch costs a few bisections, so items with few runs per stretch (a +/// part of a few wide runs, a grid that cuts every row) are walked. +fn copyItem(item: Plan.Item, landings: []const u64, sink: *CopySink) !void { + const lattice = item.lattice; + const grid = &item.destination.grid; + if (lattice.dims.len == 0 or item.first >= item.end) return walkItem(item, landings, sink); + const start = @max(item.clip_start, lattice.at(item.first).dst); + const stop = @min(item.clip_end, lattice.at(item.end - 1).dst + lattice.width); + if (start >= stop) return; + const stretches = grid.runCount(start, stop) + (stop - start) / sink.block_size + 1; + if (item.end - item.first < 16 * stretches) return walkItem(item, landings, sink); + const table = landings[item.landing_start..][0..grid.box_count]; + const source = @intFromPtr(sink.bounce.ptr) -% @as(usize, @intCast(sink.file_offset)); + var at = start; + while (at < stop) { + const box, const box_offset = grid.locate(at); + const landing: usize = @intCast(table[box] +% box_offset); + const in_block = landing % sink.block_size; + const stretch_end = @min(grid.runEnd(at, stop), at + (sink.block_size - in_block)); + // The address logical offset zero would have, were the whole + // destination laid out like this stretch. + const destination = @intFromPtr(sink.blocks[landing / sink.block_size].ptr) + in_block -% @as(usize, @intCast(at)); + const touched = @max(item.first, lattice.firstEndingAfter(at)); + const whole = @max(item.first, lattice.firstStartingFrom(at)); + const whole_end = @min(item.end, lattice.firstEndingAfter(stretch_end)); + const touched_end = @min(item.end, lattice.firstStartingFrom(stretch_end)); + if (whole < whole_end) { + try walkRuns(lattice, touched, whole, at, stretch_end, grid, table, sink); + lattice.copyRuns(whole, whole_end, destination, source); + try walkRuns(lattice, whole_end, touched_end, at, stretch_end, grid, table, sink); + } else { + try walkRuns(lattice, touched, touched_end, at, stretch_end, grid, table, sink); + } + at = stretch_end; + } +} + /// The ranges a `Scatter` gathers before it reads, as many as one direct /// read takes. const max_positional_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 else std.posix.IOV_MAX; @@ -2645,6 +2936,8 @@ test "the planner opens only local files to ask about direct I/O" { try vfs.registerBackend("remote", .{ .io = io }); const vfs_io = vfs.io(); const reads: Planner.Reads = .{ .io = vfs_io, .vfs = &vfs, .policy = .on, .alignment = 4096 }; + try std.testing.expectEqual(@as(usize, 0), reads.boundOf("remote://bucket/model.safetensors")); + try std.testing.expectEqual(@as(usize, 4096), reads.boundOf(path)); // A remote file is read exact, without an open on the planner. var remote: SourceSlot = .{ .uri = "remote://bucket/model.safetensors" }; @@ -2661,6 +2954,31 @@ test "the planner opens only local files to ask about direct I/O" { try std.testing.expectEqual(@as(usize, if (vfs.useDirectIo(file, .on)) 4096 else 0), alignment); } +test "a strided part whose source runs are not back to back counts its whole source range" { + // [32, 16] u8 at a file offset off a 64-byte alignment. + const source: safetensors.Tensor = .{ .file_uri = "wide.bin", .name = "wide", .shape = .init(.{ 32, 16 }, .u8), .offset = 100 }; + const config: Planner.Config = .{ .block_size = 256, .request_size = 512 }; + // Transposed, 512 runs of one byte: every job reads all 512 bytes, + // 576 widened to the alignment. + const transposed: LoadPart = .{ .source = &source, .lattice = .{ .width = 1, .dims = &.{ + .{ .count = 16, .src_stride = 1, .dst_stride = 32 }, + .{ .count = 32, .src_stride = 16, .dst_stride = 1 }, + } } }; + try std.testing.expectEqual(@as(backend.Backend.StridedRead, .{ .read = 512, .limit = 512 }), Planner.stridedRead(config, fixedReads(0), transposed)); + try std.testing.expectEqual(576, Planner.stridedRead(config, fixedReads(64), transposed).read); + // A bigger block calibration picked does not raise the limit past the + // smallest one's request. + const bigger_block: Planner.Config = .{ .block_size = 1024, .request_size = 1024, .gather_limit = 512 }; + try std.testing.expectEqual(512, Planner.stridedRead(bigger_block, fixedReads(0), transposed).limit); + // Every other row, in order: the gaps stretch a job's range, so the + // whole range counts, from the first row to the last. + const every_other_row: LoadPart = .{ .source = &source, .lattice = .{ .width = 16, .dims = &.{.{ .count = 16, .src_stride = 32, .dst_stride = 16 }} } }; + try std.testing.expectEqual(15 * 32 + 16, Planner.stridedRead(config, fixedReads(0), every_other_row).read); + // Rows back to back read within their job's range. + const rows: LoadPart = .{ .source = &source, .lattice = .{ .width = 16, .dims = &.{.{ .count = 32, .src_stride = 16, .dst_stride = 16 }} } }; + try std.testing.expectEqual(0, Planner.stridedRead(config, fixedReads(64), rows).read); +} + test "source planner coalesces exact adjacent and overlapping tensor ranges per file" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -4087,3 +4405,61 @@ test "column-sharded tensors load as one DMA piece per device and block, read ex try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); } } + +test "a strided part of a remote file is checked against its exact source range" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + var bytes: [128 * 128]u8 = undefined; + testing.pattern(&bytes, 1); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "square.safetensors", ®istry, &.{ + .{ .name = "square", .dtype = "U8", .dims = &.{ 128, 128 }, .bytes = &bytes }, + }); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + // The same file through a registered scheme: a remote file, read exact. + try vfs.registerBackend("remote", .{ .io = io }); + const local = registry.tensors.get("square").?; + var remote = local; + var uri_buffer: [1100]u8 = undefined; + remote.file_uri = try std.fmt.bufPrint(&uri_buffer, "remote://{s}", .{local.file_uri}); + std.debug.assert(local.offset % 4096 != 0); + + var transposed: [128 * 128]u8 = undefined; + for (0..128) |column| { + for (0..128) |row| transposed[column * 128 + row] = bytes[row * 128 + column]; + } + // Transposed: its source read whole by every job, a request exactly, + // at an offset off the direct I/O alignment. + const dims = [_]backend.Lattice.Dim{ + .{ .count = 128, .src_stride = 1, .dst_stride = 128 }, + .{ .count = 128, .src_stride = 128, .dst_stride = 1 }, + }; + const loader = try Loader.create(allocator, vfs.io(), devices.platform, .{ .read_parallelism = 2 }); + defer loader.destroy(); + try std.testing.expectEqual(VFS.direct_io_alignment, loader.reads.alignment); + loader.plan_config.request_size = bytes.len; + + var output: Buffer = undefined; + const part: LoadPart = .{ .source = &remote, .lattice = .{ .width = 1, .dims = &dims } }; + const fit = loader.stridedRead(part); + try std.testing.expectEqual(fit.limit, fit.read); + const spec: LoadSpec = .{ .shape = .init(.{ 128, 128 }, .u8), .output = &output, .parts = (&part)[0..1] }; + try loader.awaitBatch(try loader.submit(&.{spec}, .none)); + defer output.deinit(); + try testing.expectShards(allocator, io, &output, &transposed, .default); + + // A local file may be read directly: its range widened to the alignment + // no longer fits. + if (VFS.direct_io_alignment != 0) { + const local_part: LoadPart = .{ .source = &local, .lattice = .{ .width = 1, .dims = &dims } }; + const widened = loader.stridedRead(local_part); + try std.testing.expect(widened.read > widened.limit); + } +} diff --git a/zml/io/loader.zig b/zml/io/loader.zig index 78b80cdb..71605eb0 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -16,8 +16,9 @@ //! checkpoint tensor is one part. //! A strided part of at most `max_part_runs` runs, or of runs of at least //! `large_run_bytes`, becomes one part per run; a spec with any other -//! strided part (row interleaves, transposes, broadcasts) is refused. The -//! parts go in destination order, and the bytes none covers are zero. Each +//! strided part (row interleaves, transposes, broadcasts) goes to the +//! backend as strided parts, which the direct backend gathers. The parts go +//! in destination order, and the bytes none covers are zero. Each //! `LoadSpec` is one destination, which the backends trust as //! `backend.LoadSpec` states. Only `Source` makes bindings and its pieces //! are disjoint by construction: nothing here checks that (Source's tests @@ -251,13 +252,15 @@ pub const Loader = struct { /// Every tensor and program input is checked once, before anything is /// published: a tensor `store` did not bind, a source that reads no /// bytes (an empty checkpoint tensor among them), a source that this - /// loader cannot assemble (a run that starts or ends inside a byte, or - /// strided runs that do not expand into contiguous ones: more than - /// `max_part_runs` runs, each shorter than `large_run_bytes`, as a - /// transpose, a broadcast or a row interleave makes), a sub-byte last - /// dimension that does not pack into whole bytes, a placement its mesh - /// does not divide or on devices the platform lacks, and a memory kind - /// the platform cannot take are `error.InvalidBinding`. + /// loader cannot assemble (a run that starts or ends inside a byte, + /// strided parts into host memory, a slice along an inner axis, a + /// broadcast or a transpose whose source range does not fit the direct + /// backend's smallest read: the checkpoint's read chunk (8 MiB locally), + /// rounded up to whole blocks of the smallest it may pick, 2 MiB or a + /// pinned `dma_block_size`), a sub-byte last dimension that does not + /// pack into whole bytes, a placement its mesh does not divide or on + /// devices the platform lacks, and a memory kind the platform cannot + /// take are `error.InvalidBinding`. /// /// `buffers` is `mem.bufferize` of `model`, whose buffers are empty. A /// tree of another structure (other union arms, optionals or slice @@ -297,7 +300,7 @@ pub const Loader = struct { try self.awaitSetup(); // The load's time starts once the set-up is done. self.stats.begin(self.io); - var prepared = try prepareModel(self.allocator, self.platform, store, model, buffers); + var prepared = try prepareModel(self.allocator, self.platform, self.backend.?, store, model, buffers); errdefer prepared.arena.deinit(); var items = prepared.specs.len; for (prepared.computed) |item| items += item.compute.inputs.len; @@ -394,7 +397,7 @@ pub const Loader = struct { for (item.compute.inputs, inputs, 0..) |source, *input, index| { input.* = .{ ._platform = self.platform, ._shape = source.shape, ._shards = .empty }; const target: Target = .{ .path = item.path, .shape = item.shape, .input = .{ .index = index, .shape = source.shape } }; - const spec = try prepareSpec(arena.allocator(), self.platform, target, source, .default, input); + const spec = try prepareSpec(arena.allocator(), self.platform, self.backend.?, target, source, .default, input); try specs.append(arena.allocator(), spec); } const compiled = try self.compiledProgram(store, item.compute, item.shape); @@ -700,6 +703,7 @@ const PreparedModel = struct { fn prepareModel( allocator: std.mem.Allocator, platform: *const Platform, + backend_: Backend, store: *const TensorStore, model: anytype, buffers: *Bufferized(@TypeOf(model.*)), @@ -731,7 +735,7 @@ fn prepareModel( return error.InvalidBinding; }; switch (binding) { - .load => |load| specs.appendAssumeCapacity(try prepareSpec(arena.allocator(), platform, target, load.source, load.memory, pair.buffer)), + .load => |load| specs.appendAssumeCapacity(try prepareSpec(arena.allocator(), platform, backend_, target, load.source, load.memory, pair.buffer)), .compute => |compute| { try checkPlacement(platform, target, compute.program.name()); computed.appendAssumeCapacity(.{ .path = pair.path, .shape = tensor.shape(), .output = pair.buffer, .compute = compute }); @@ -744,7 +748,9 @@ fn prepareModel( /// A strided part with few runs, or with large ones, expands into one part /// per run, each read and transferred on its own: at most `max_part_runs` /// runs, or runs of at least `large_run_bytes`. A spec with any other -/// strided part is refused. +/// strided part goes to the backend whole, as strided parts +/// (`backend.Lattice`), which the direct backend reads by destination +/// ranges. const max_part_runs = 64; const large_run_bytes = 1 << 20; @@ -774,13 +780,15 @@ const Target = struct { /// The spec filling `output`, the buffer of `target`, from `source` into /// `memory`, its parts in `arena`: the source lowered to bytes -/// (`Source.Placed.lower`), expanded into its runs, in destination order. -/// The one check of what the backends trust (`backend.LoadSpec`); a rule it -/// breaks is `InvalidBinding`, logged with the target, and with the key its -/// source reads once the source is lowered. +/// (`Source.Placed.lower`), expanded into its runs unless a part does not +/// expand, in destination order. The one check of what the backends trust +/// (`backend.LoadSpec`, `Backend.stridedRead`); a rule it breaks is +/// `InvalidBinding`, logged with the target, and with the key its source +/// reads once the source is lowered. fn prepareSpec( arena: std.mem.Allocator, platform: *const Platform, + backend_: Backend, target: Target, source: TensorStore.Source.Placed, memory: Memory, @@ -799,20 +807,23 @@ fn prepareSpec( return error.InvalidBinding; } const key = lowered[0].source.name; + var expands = true; + var runs: u64 = 0; + for (lowered) |part| { + expands = expands and expandable(part.lattice); + runs += part.lattice.runs(); + } try checkPlacement(platform, target, key); if (!backend.supportsMemory(platform, memory)) { load_log.err("{f} from {s} is bound to {t} memory, which {t} cannot take", .{ target, key, memory, platform.target }); return error.InvalidBinding; } - var runs: u64 = 0; - for (lowered) |part| { - if (!expandable(part.lattice)) { - load_log.err("{f} from {s}: a part of {d} strided runs of {d} bytes (a transpose, a broadcast or an interleave), more than {d} runs under {d} bytes, which this loader does not assemble", .{ target, part.source.name, part.lattice.runs(), part.lattice.width, max_part_runs, large_run_bytes }); - return error.InvalidBinding; - } - runs += part.lattice.runs(); + if (!expands and memory.isHost()) { + load_log.err("{f} from {s}: strided parts into {t} memory, which this loader does not assemble", .{ target, key, memory }); + return error.InvalidBinding; } - const parts = if (runs == lowered.len) lowered else expanded: { + const parts = if (!expands) lowered else expanded: { + if (runs == lowered.len) break :expanded lowered; const expanded = try arena.alloc(LoadPart, @intCast(runs)); var next: usize = 0; for (lowered) |part| { @@ -832,6 +843,14 @@ fn prepareSpec( load_log.err("{f} from {s}: its parts leave their sources or its {d} bytes, pass the largest file offset, or come out of order", .{ target, key, shape.byteSize() }); return error.InvalidBinding; } + // After `checkParts`, which bounds the offsets `stridedRead` adds. + for (parts) |part| { + const strided = backend_.stridedRead(part); + if (strided.read > strided.limit) { + load_log.err("{f} from {s}: a strided part spans {d} source bytes, more than one read of {d}", .{ target, key, strided.read, strided.limit }); + return error.InvalidBinding; + } + } return .{ .shape = shape, .memory = memory, .output = output, .parts = parts }; } @@ -1433,9 +1452,8 @@ test "a rejected binding or a program of the wrong output fails the loader" { const splits = platform.devices.len > 1 and 4 % platform.devices.len == 0; // No binding, an empty source or program input, a mesh of other - // devices, a host kind the platform cannot take, a broadcast of more - // strided runs than expand, and a program of the wrong shape or - // placement. + // devices, a host kind the platform cannot take, and a program of the + // wrong shape or placement. const Case = struct { tensor: Tensor, at_wait: bool = false, skip: bool = false }; const cases = [_]Case{ .{ .tensor = Tensor.fromShape(fixture.value.shape()) }, @@ -1445,7 +1463,6 @@ test "a rejected binding or a program of the wrong output fails the loader" { .{ .tensor = foreign[1] }, .{ .tensor = fixture.computed("value", foreign[2], LoaderTestFixture.Identity{}) }, .{ .tensor = pinned, .skip = backend.supportsMemory(platform, .host_pinned) }, - .{ .tensor = fixture.store.view().source("value", .{.x}).broad(Shape.init(.{ .y = max_part_runs + 1, .x = 4 }, .u8)).withPartitioning(.replicated, .{}).bind() }, .{ .tensor = fixture.computed("value", fixture.value, Rows{}), .at_wait = true }, .{ .tensor = fixture.computed("value", misplaced, LoaderTestFixture.Identity{}), .at_wait = true, .skip = !splits }, }; @@ -1520,9 +1537,9 @@ fn expectStructureMismatch(fixture: *LoaderTestFixture, comptime Model: type, bu /// Two safetensors files for composite loads, every row 16 bytes (8 u16): /// `first` holds `a` [3, 8] then `b` [2, 8], adjacent, then `spacer`, then /// `q` [8, 8], `k` [4, 8] and `v` [4, 8]; `second` holds `plain` [4, 8], -/// then `c` [4, 8] and the experts `e{0..3}.w1` and `e{0..3}.w3` [2, 8]. -/// Pinned after `init`. Each store takes a copy of `registry` -/// (`testing.storeOf`). +/// then `c` [4, 8], the experts `e{0..3}.w1` and `e{0..3}.w3` [2, 8] and +/// `wide` [64, 8]. `empty` [0, 8] is registered without bytes. Pinned after +/// `init`. Each store takes a copy of `registry` (`testing.storeOf`). const CompositeFixture = struct { tmp: std.testing.TmpDir, registry: safetensors.TensorRegistry, @@ -1535,6 +1552,7 @@ const CompositeFixture = struct { v: [64]u8, w1: [4][32]u8, w3: [4][32]u8, + wide: [1024]u8, const expert_names = [2][4][]const u8{ .{ "e0.w1", "e1.w1", "e2.w1", "e3.w1" }, @@ -1559,6 +1577,7 @@ const CompositeFixture = struct { } var spacer: [16]u8 = undefined; testing.pattern(&spacer, 5); + testing.pattern(&self.wide, 6); try testing.writeSafetensors(allocator, io, self.tmp.dir, "first.safetensors", &self.registry, &.{ .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &self.a }, .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.b }, @@ -1578,6 +1597,13 @@ const CompositeFixture = struct { .{ .name = expert_names[1][2], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[2] }, .{ .name = expert_names[0][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[3] }, .{ .name = expert_names[1][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[3] }, + .{ .name = "wide", .dtype = "U16", .dims = &.{ 64, 8 }, .bytes = &self.wide }, + }); + try self.registry.registerTensor(.{ + .file_uri = self.registry.tensors.get("a").?.file_uri, + .name = "empty", + .shape = .init(.{ 0, 8 }, .u16), + .offset = 0, }); } @@ -1752,7 +1778,228 @@ test "loader assembles interleaved, stacked and split sources" { } } -test "loader fills gaps, reordered and strided rows" { +test "loader gathers strided sources: a row interleave across files, broadcasts and a transpose" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const Source = TensorStore.Source; + // Rows of 16 bytes: g and u interleave row by row, 96 runs each; s0 and + // s1 repeat 100 and 60 times; z [24, 4] is transposed, 96 runs of two + // bytes, then padded with two zero rows. Every part has more runs than + // expand, so the direct backend gathers them, its jobs reading g's rows + // from one file and u's from the other. + var g: [96 * 16]u8 = undefined; + var u: [96 * 16]u8 = undefined; + var z: [24 * 8]u8 = undefined; + var s0: [4]u8 = undefined; + var s1: [4]u8 = undefined; + testing.pattern(&g, 1); + testing.pattern(&u, 2); + testing.pattern(&z, 3); + testing.pattern(&s0, 4); + testing.pattern(&s1, 5); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "first.safetensors", ®istry, &.{ + .{ .name = "g", .dtype = "U16", .dims = &.{ 96, 8 }, .bytes = &g }, + .{ .name = "s0", .dtype = "F32", .dims = &.{}, .bytes = &s0 }, + .{ .name = "z", .dtype = "U16", .dims = &.{ 24, 4 }, .bytes = &z }, + }); + try testing.writeSafetensors(allocator, io, tmp.dir, "second.safetensors", ®istry, &.{ + .{ .name = "s1", .dtype = "F32", .dims = &.{}, .bytes = &s1 }, + .{ .name = "u", .dtype = "U16", .dims = &.{ 96, 8 }, .bytes = &u }, + }); + + var interleaved: [192 * 16]u8 = undefined; + for (0..96) |r| { + @memcpy(interleaved[2 * r * 16 ..][0..16], g[r * 16 ..][0..16]); + @memcpy(interleaved[(2 * r + 1) * 16 ..][0..16], u[r * 16 ..][0..16]); + } + var scales: [160 * 4]u8 = undefined; + for (0..160) |i| @memcpy(scales[i * 4 ..][0..4], if (i < 100) &s0 else &s1); + var transposed: [6 * 48]u8 = @splat(0); + for (0..4) |column| { + for (0..24) |row| @memcpy(transposed[column * 48 + row * 2 ..][0..2], z[row * 8 + column * 2 ..][0..2]); + } + + const Model = struct { by_rows: Tensor, by_columns: Tensor, scales_by_rows: Tensor, scales_by_columns: Tensor, transposed: Tensor, transposed_by_columns: Tensor }; + for ([_]usize{ 1, 2, 4 }) |device_count| { + for ([_]usize{ 0, 64 }) |alignment| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, ®istry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + const view = store.view(); + const rows = Source.stack(&.{ view.source("g", .{ .row, .col }), view.source("u", .{ .row, .col }) }, 1, .proj) + .merge(.{ .row = .{ .row, .proj } }); + const repeated = Source.concatenate(&.{ + view.source("s0", .{}).broad(Shape.init(.{ .c = 100 }, .f32)), + view.source("s1", .{}).broad(Shape.init(.{ .c = 60 }, .f32)), + }, .c).splitAxis(.c, .{ .row = 40, .col = 4 }); + const flipped = view.source("z", .{ .row, .col }).transpose(.{ .col, .row }).withTags(.{ .row, .col }) + .pad(0, .{ .row = Tensor.Pad{ .high = 2 } }); + const model: Model = .{ + .by_rows = rows.withPartitioning(.tp, .{ .row = .model }).bind(), + .by_columns = rows.withPartitioning(.tp, .{ .col = .model }).bind(), + .scales_by_rows = repeated.withPartitioning(.tp, .{ .row = .model }).bind(), + .scales_by_columns = repeated.withPartitioning(.tp, .{ .col = .model }).bind(), + .transposed = flipped.withPartitioning(.replicated, .{}).bind(), + .transposed_by_columns = flipped.withPartitioning(.tp, .{ .col = .model }).bind(), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, devices.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + // Jobs of 128 bytes: every destination takes several. + try loader.awaitSetup(); + switch (loader.backend.?) { + .direct => |direct| { + direct.plan_config.request_size = 256; + direct.reads.alignment = alignment; + }, + .buffered => {}, + } + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.by_rows, &interleaved, .default); + try testing.expectShards(allocator, io, &buffers.by_columns, &interleaved, .default); + try testing.expectShards(allocator, io, &buffers.scales_by_rows, &scales, .default); + try testing.expectShards(allocator, io, &buffers.scales_by_columns, &scales, .default); + try testing.expectShards(allocator, io, &buffers.transposed, &transposed, .default); + try testing.expectShards(allocator, io, &buffers.transposed_by_columns, &transposed, .default); + } + } +} + +test "loader gathers a block-scale swizzle, its padding zero" { + const allocator = std.testing.allocator; + const io = std.testing.io; + // Scales [e=4, m=31, k=6] in the 128x4 layout at 8x4: m padded to 32 + // and tiled by 2x4, k padded to 8. Its parts repeat runs of four bytes + // or two, more than expand, so the direct backend gathers them. + var scales: [4 * 31 * 6]u8 = undefined; + testing.pattern(&scales, 7); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "scales.safetensors", ®istry, &.{ + .{ .name = "scales", .dtype = "U8", .dims = &.{ 4, 31, 6 }, .bytes = &scales }, + }); + var swizzled: [4 * 32 * 8]u8 = @splat(0); + for (0..4) |e| for (0..4) |mo| for (0..2) |ko| for (0..4) |mi| for (0..2) |m4| for (0..4) |ki| { + const m = mo * 8 + m4 * 4 + mi; + const k = ko * 4 + ki; + const flat = (((mo * 2 + ko) * 4 + mi) * 2 + m4) * 4 + ki; + if (m < 31 and k < 6) swizzled[e * 256 + flat] = scales[(e * 31 + m) * 6 + k]; + }; + + const Model = struct { replicated: Tensor, by_rows: Tensor }; + for ([_]usize{ 1, 2, 4 }) |device_count| { + for ([_]usize{ 0, 64 }) |alignment| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, ®istry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + const swizzle = store.view().source("scales", .{ .e, .m, .k }) + .pad(0, .{ .m = Tensor.Pad{ .high = 1 }, .k = Tensor.Pad{ .high = 2 } }) + .splitAxis(.m, .{ .mo = -1, .m4 = 2, .mi = 4 }) + .splitAxis(.k, .{ .ko = -1, .ki = 4 }) + .transpose(.{ .e, .mo, .ko, .mi, .m4, .ki }) + .merge(.{ .flat = .{ .mo, .ko, .mi, .m4, .ki } }); + const model: Model = .{ + .replicated = swizzle.withPartitioning(.replicated, .{}).bind(), + .by_rows = swizzle.withPartitioning(.tp, .{ .e = .model }).bind(), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, devices.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + // Jobs of at most 1024 bytes, each reading the 744 source + // bytes whole: the swizzle reorders its runs. + try loader.awaitSetup(); + switch (loader.backend.?) { + .direct => |direct| { + direct.plan_config.request_size = 1024; + direct.reads.alignment = alignment; + }, + .buffered => {}, + } + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.replicated, &swizzled, .default); + try testing.expectShards(allocator, io, &buffers.by_rows, &swizzled, .default); + } + } +} + +test "bindings the loader cannot fill fail it before submission" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // Three nibbles: whole bytes hold no element boundary past the first two. + try fixture.registry.registerTensor(.{ + .file_uri = fixture.registry.tensors.get("a").?.file_uri, + .name = "nibbles", + .shape = .init(.{3}, .u4), + .offset = 0, + }); + try fixture.registry.registerTensor(.{ + .file_uri = fixture.registry.tensors.get("a").?.file_uri, + .name = "nibble_rows", + .shape = .init(.{ 2, 3 }, .f4e2m1), + .offset = 0, + }); + const platform = testing.env(); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + const Source = TensorStore.Source; + const view = store.view(); + const rows = CompositeFixture.rows; + + const rejected = [_]Tensor{ + // Nothing to read: an empty tensor padded. + rows(view, "empty").pad(0, .{ .row = Source.Pad{ .high = 8 } }).withPartitioning(.replicated, .{}).bind(), + // Twelve bits: its one run ends inside a byte (no known checkpoint + // has one). + view.createReplicatedTensor("nibbles", null), + // Strided parts into host memory. + host: { + const tensor = rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{}).bind(); + view.setMemory(tensor, .host_pinned); + break :host tensor; + }, + // Three whole bytes, in rows of three nibbles that do not pack, as a + // tensor and as a program input. + view.createReplicatedTensor("nibble_rows", null), + Source.bindProgram(&.{view.source("nibble_rows", null).withPartitioning(.replicated, .{})}, .init(.{3}, .u8), LoaderTestFixture.Identity{}), + // A program input whose transpose reads its 1024 source bytes in + // every job, more than one read. + Source.bindProgram(&.{rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{})}, .init(.{ 8, 64 }, .u16), LoaderTestFixture.Identity{}), + }; + var output: Buffer = undefined; + for (&rejected) |*tensor| { + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + // One read holds half of `wide`; the buffered backend reads any part. + try loader.awaitSetup(); + switch (loader.backend.?) { + .direct => |direct| direct.plan_config.request_size = 512, + .buffered => if (tensor == &rejected[rejected.len - 1]) continue, + } + try std.testing.expectError(error.InvalidBinding, loader.start(tensor, &output, &store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + } +} + +test "loader fills gaps, reordered and strided rows, and a transpose" { const allocator = std.testing.allocator; const io = std.testing.io; var fixture: CompositeFixture = undefined; @@ -1768,9 +2015,10 @@ test "loader fills gaps, reordered and strided rows" { const a = rows(view, "a"); // b's second row past a gap, the rest zero; b then a, each padded; a's - // last row before its first two, one source feeding two parts; and a's - // rows every other row, a strided part of few runs, expanded. - const Model = struct { sparse: Tensor, padded: Tensor, swapped: Tensor, strided: Tensor }; + // last row before its first two, one source feeding two parts; a's + // rows every other row, a strided part of few runs, expanded; and a + // transpose, 512 runs of two bytes, gathered. + const Model = struct { sparse: Tensor, padded: Tensor, swapped: Tensor, strided: Tensor, transposed: Tensor }; const model: Model = .{ .sparse = rows(view, "b").slice(.row, .{ .start = 1 }).pad(0, .{ .row = Pad{ .low = 4, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), .padded = Source.concatenate(&.{ @@ -1780,6 +2028,7 @@ test "loader fills gaps, reordered and strided rows" { .swapped = Source.concatenate(&.{ a.slice(.row, .{ .start = 2 }), a.slice(.row, .{ .end = 2 }) }, .row) .pad(0, .{ .row = Pad{ .high = 5 } }).withPartitioning(.replicated, .{}).bind(), .strided = a.pad(0, .{ .row = Pad{ .interior = 1, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), + .transposed = rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{}).bind(), }; var buffers = try mem.bufferize(allocator, &model); defer mem.deinitBufferized(allocator, &buffers); @@ -1802,6 +2051,11 @@ test "loader fills gaps, reordered and strided rows" { expected = @splat(0); for (0..3) |i| @memcpy(expected[i * 32 ..][0..16], fixture.a[i * 16 ..][0..16]); try testing.expectShards(allocator, io, &buffers.strided, &expected, .default); + var transposed_bytes: [1024]u8 = undefined; + for (0..8) |column| { + for (0..64) |row| @memcpy(transposed_bytes[column * 128 + row * 2 ..][0..2], fixture.wide[row * 16 + column * 2 ..][0..2]); + } + try testing.expectShards(allocator, io, &buffers.transposed, &transposed_bytes, .default); } test "a program takes its inputs from sources, in order" { From 10fb8e0921c55d8f543d709d252ff9de77aa16d6 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 15:58:40 +0100 Subject: [PATCH 085/103] zml/io: deliver host_pinned destinations in place The direct backend now takes tensors bound to host_pinned memory, and no DMA goes to them: the worker that reads a host destination's bytes writes them into its PJRT host buffers. A host job's read goes straight into the destination ranges when it has no alignment padding and its transfers address each byte once, in order; a read widened for direct I/O, or one that overlapping parts read twice, lands in the request's blocks and is copied from there. The bytes no part covers are zeroed in place by a host fill job. Host and DMA parts never share a job, and host jobs take no DMA events. Each device of a replicated host destination holds a buffer of its own, written from the first device's. A first pinned allocation is slow on some drivers, so a task of the worker group creates a batch's host destinations ahead of their reads, in claim order, holding one completion unit of the batch; a worker that finds its destination missing still creates it. The batch planning and summary debug lines count host jobs and the time spent ahead. supportsMemory accepts host_pinned where the platform loads directly and every device has a pinned host space; host_mapped stays refused. The composite replays, the failure loop and the loader's memory-kind test run host_pinned on CPU, and a new test loads rows, columns, replicas, an overlap and a device tensor from one file, read exact and widened. load_check binds every tensor to ZML_LOAD_MEMORY. Co-Authored-By: Claude Opus 5.5 --- examples/io/load_check.zig | 9 + zml/io/backend.zig | 20 +- zml/io/direct_loader.zig | 626 +++++++++++++++++++++++++++++++------ 3 files changed, 550 insertions(+), 105 deletions(-) diff --git a/examples/io/load_check.zig b/examples/io/load_check.zig index 5ef8e9e0..9c8f68cd 100644 --- a/examples/io/load_check.zig +++ b/examples/io/load_check.zig @@ -35,6 +35,10 @@ //! (`Loader.Options.dma_block_size`); unset, calibration picks one. //! - `ZML_DIRECT_IO=off|on|auto`: direct I/O for local checkpoint files //! (`Loader.Options.direct_io`, `VFS.DirectIo`); `auto` when unset. +//! - `ZML_LOAD_MEMORY=default|host_pinned|host_mapped`: where every tensor +//! is delivered (`TensorStore.Memory`, `View.setMemory`); `default`, device +//! memory, when unset. The loader refuses a kind the platform cannot take +//! (`InvalidBinding`). //! - `ZML_GPU_MEMORY_FRACTION=f`: a preallocated BFC pool of that fraction //! of each device's memory on CUDA, ROCm and oneAPI, instead of the //! platform's allocator defaults, for a run that observes the loader's @@ -75,6 +79,10 @@ pub fn main(init: std.process.Init) !void { return error.InvalidArgument; }; const dma_block_mib = try envNumber(u32, init.environ_map, "ZML_DMA_BENCH_BLOCK_MIB"); + const memory = std.meta.stringToEnum(zml.io.next.TensorStore.Memory, init.environ_map.get("ZML_LOAD_MEMORY") orelse "default") orelse { + log.err("ZML_LOAD_MEMORY must be default, host_pinned or host_mapped", .{}); + return error.InvalidArgument; + }; var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); @@ -125,6 +133,7 @@ pub fn main(init: std.process.Init) !void { view.createTensor(key, null, .tp, columns) else view.createTensor(key, null, .tp, rows); + if (memory != .default) view.setMemory(tensor.*, memory); } var progress = std.Progress.start(io, .{ .root_name = "load_check", .disable_printing = true }); diff --git a/zml/io/backend.zig b/zml/io/backend.zig index 2cc96fd4..50b6f390 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -393,12 +393,22 @@ pub fn loadsDirect(platform: *const Platform) bool { }; } -/// Whether the loader can deliver into `kind` on `platform`: device memory, -/// on every platform. The host kinds need a backend that writes host -/// memory in place. +/// Whether the loader can deliver into `kind` on `platform`. `host_pinned` +/// needs the direct pipeline and a pinned host memory space on every +/// device. CPU loads through the direct backend, so it accepts it (its +/// plugin resolves every kind to its one memory space). `host_mapped` +/// needs views of host memory in device memory, which no backend makes. pub fn supportsMemory(platform: *const Platform, kind: Memory) bool { - _ = platform; - return kind == .default; + switch (kind) { + .default => return true, + .host_pinned => {}, + .host_mapped => return false, + } + if (!loadsDirect(platform)) return false; + for (platform.devices) |*device| { + if (device.memory(.host_pinned) == null) return false; + } + return true; } /// The transfer implementation selected for the platform (`loadsDirect`). diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 9b0da89f..9d1f3cd7 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -59,6 +59,17 @@ //! worker time; as programs it copied 1 to 3 GiB and loaded 0.1 to 1.1 s //! faster, for 4 to 6 GiB more device memory at the peak. //! +//! Host memory. A destination in a host memory space (`Memory.isHost`) +//! takes no DMA: the worker that reads its bytes writes them in place into +//! its PJRT host buffers, which an allocation task creates ahead of the +//! reads (`Loader.startAllocations`). A host job reads straight into its +//! destinations when its read is exact and addresses each byte once, in +//! order; a widened read, or parts that overlap, goes through its slot's +//! blocks and is copied from there. Bytes no part covers are zeroed in +//! place. Each device of a replicated destination holds a copy of its own, +//! which the worker writes from the first one. Host and DMA parts never +//! share a job, and no strided part goes to host memory. +//! //! Locks. Where several pumps are held, they are taken in device order. A //! pump's mutex may be held while a released request gives its credit back //! (`Credits`), never the reverse, and no other lock is taken under the @@ -89,8 +100,8 @@ const load_log = @import("log.zig").load; const testing = @import("../testing.zig"); /// Creating a tensor's device state: host memory or PJRT, since the front -/// end checked its placement. `LazyOnce` replays it to every later reader -/// of the tensor. +/// end checked its placement and memory kind. `LazyOnce` replays it to every +/// later reader of the tensor. const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; /// Bounds per-device event overhead for tiny tensors. This is not a measured @@ -268,11 +279,12 @@ pub const Loader = struct { /// batch: work on the first file starts while the rest is planned. The /// batch owns its destinations and parts until `awaitBatch` retires it. /// The specs keep the `LoadSpec` contract and `stridedRead`'s limit, - /// which the front end checked, and this backend's: device memory. A - /// planning failure fails the loader (a partial submission can never - /// complete): the batch is awaited here and the caller sees only the - /// first error. A failed loader refuses every submit with its first - /// error. + /// which the front end checked. A planning failure fails the loader (a + /// partial submission can never complete): the batch is awaited here + /// and the caller sees only the first error. A failed loader refuses + /// every submit with its first error. The batch's host-resident + /// destinations are then created ahead of their reads + /// (`startAllocations`). pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { try self.checkOpen(); const batch = batch: { @@ -294,6 +306,7 @@ pub const Loader = struct { // Stamped before the sentinel goes, so before the batch can complete. batch.sealed_at = .now(self.io, .awake); self.batch_count += 1; + self.startAllocations(batch); // Every plan is visible: drop the publish sentinel. A batch without // jobs completes right here. batch.releaseUnits(1); @@ -306,6 +319,56 @@ pub const Loader = struct { return Planner.stridedRead(self.plan_config, self.reads, part); } + /// Creates the batch's host-resident destinations ahead of their reads, + /// in claim order, on a task of its own. A first-time pinned allocation + /// runs at about 14-15 GiB/s on four B70 (oneAPI), serialized whatever + /// the number of callers (one to sixteen threads measured alike; the + /// time is the xe driver's VM_BIND), so on the workers' read path it + /// stalled them for 10-12 ms per tensor; ahead of them it overlaps the + /// reads. A worker that still finds its destination missing creates it + /// itself (`LazyOnce`). The task holds one completion unit, so the batch + /// outlives it; without a task the workers create everything. + fn startAllocations(self: *Loader, batch: *Batch) void { + var host_destinations: usize = 0; + for (batch.destinations) |*destination| host_destinations += @intFromBool(destination.memory.isHost()); + if (host_destinations == 0) return; + _ = batch.remaining.fetchAdd(1, .acq_rel); + self.worker_group.concurrent(self.io, allocateMain, .{ self, batch, host_destinations }) catch |err| { + load_log.debug("batch {d}: no allocation task, workers create its {d} host destinations: {s}", .{ batch.sequence, host_destinations, @errorName(err) }); + batch.releaseUnits(1); + }; + } + + /// Walks the host jobs in claim order (the plans in publish order, each + /// plan's jobs in order) and their transfers as `run` does, skipping a + /// destination already created. `ensured` also counts one a worker was + /// still creating, which the task waited for. + fn allocateMain(self: *Loader, batch: *Batch, host_destinations: usize) void { + var ensured: usize = 0; + var previous: ?*Destination = null; + walk: for (batch.plans.items) |plan| for (plan.jobs) |job| { + switch (job.kind) { + .host_direct, .host_bounce, .host_fill => {}, + .dma_read, .dma_land, .dma_fill => continue, + } + for (plan.transfersOf(job)) |transfer| { + const destination = transfer.destination; + if (destination == previous or destination.state.readyValue() != null) continue; + previous = destination; + if (self.first_error.get() != null) break :walk; + const started = awakeNs(self.io); + _ = destination.ensureState(self) catch |err| { + self.recordError(err, "initialize tensor {s} from {s} ahead of its reads, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); + break :walk; + }; + _ = self.metrics.ahead_init_ns.fetchAdd(awakeNs(self.io) -| started, .monotonic); + ensured += 1; + } + }; + load_log.debug("batch allocations: batch={d}, host_destinations={d}, ensured={d}, done=+{d:.3}s", .{ batch.sequence, host_destinations, ensured, secondsBetween(self.created_at, .now(self.io, .awake)) }); + batch.releaseUnits(1); + } + /// Waits for the batch's last completion unit, retires it and returns /// the loader's sticky error if the loader failed. Destroying the batch /// destroys its transfer managers, and PJRT's manager destructor errors @@ -319,7 +382,8 @@ pub const Loader = struct { /// completed without an error submitted every byte of every target (the /// pump flagged the submission that completed its bytes); one that did /// not would leave a buffer PJRT then errors, a bookkeeping bug this - /// panics on. + /// panics on. A host-resident destination has no target: its workers + /// write it in place, which only the debug check of its progress covers. pub fn awaitBatch(self: *Loader, batch: *Batch) !void { batch.done.waitUncancelable(self.io); const done_at: std.Io.Timestamp = .now(self.io, .awake); @@ -384,7 +448,8 @@ pub const Loader = struct { /// Creates the batch's destinations, one per spec, and their parts in /// its arena, their sources and dimensions staying the specs' until the /// batch is awaited (`backend.LoadSpec`). A destination's device state - /// is created lazily by the first worker that reads or fills for it. + /// is created lazily by the first worker that reads or fills for it, + /// or by the allocation task. fn createDestinations( self: *Loader, batch: *Batch, @@ -398,7 +463,6 @@ pub const Loader = struct { const parts = try arena.alloc(Part, part_count); var next_part: usize = 0; for (specs, destinations) |spec, *destination| { - std.debug.assert(spec.memory == .default); // submit const grid: ShardGrid = .init(self.platform, spec.shape); const destination_parts = parts[next_part..][0..spec.parts.len]; next_part += spec.parts.len; @@ -416,9 +480,11 @@ pub const Loader = struct { const strided = self.stridedRead(part); std.debug.assert(strided.read <= strided.limit); // backend.Backend.stridedRead } + std.debug.assert(!gathered or !spec.memory.isHost()); // backend.LoadSpec.memory destination.* = .{ .shape = spec.shape, .grid = grid, + .memory = spec.memory, .output = spec.output, .parts = destination_parts, .gathered = gathered, @@ -550,7 +616,8 @@ pub const Loader = struct { if (job.kind == .dma_land and plan.transfersOf(job)[0].destination.gathered) gather_jobs += 1; } } - const fill_jobs = kinds.get(.dma_fill); + const fill_jobs = kinds.get(.dma_fill) + kinds.get(.host_fill); + const host_jobs = kinds.get(.host_direct) + kinds.get(.host_bounce) + kinds.get(.host_fill); const source_jobs = jobs - fill_jobs; const average_read_size = if (source_jobs == 0) 0 else source_bytes / source_jobs; const coalescing_ratio = if (source_jobs == 0) @@ -558,13 +625,14 @@ pub const Loader = struct { else @as(f64, @floatFromInt(batch.parts.len)) / @as(f64, @floatFromInt(source_jobs)); - load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, staged_jobs={d}, gather_jobs={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ + load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, host_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, staged_jobs={d}, gather_jobs={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ batch.sequence, batch.plans.items.len, @as(f64, @floatFromInt(planning_ns)) / std.time.ns_per_s, source_bytes, source_jobs, fill_jobs, + host_jobs, batch.parts.len, transfers, dma_submissions, @@ -589,11 +657,12 @@ pub const Loader = struct { self.credits.count(), self.workspace.mappedBytes(), }); - load_log.debug("loader waits: credit_wait_ms_per_read={d:.3}, read_ms_per_read={d:.3}, dma_stage_ms_per_read={d:.3}, tensor_init_ms_per_read={d:.3}", .{ + load_log.debug("loader waits: credit_wait_ms_per_read={d:.3}, read_ms_per_read={d:.3}, dma_stage_ms_per_read={d:.3}, tensor_init_ms_per_read={d:.3}, ahead_init_ms={d:.3}", .{ millisecondsPer(self.metrics.credit_wait_ns.load(.acquire), reads), millisecondsPer(self.metrics.read_ns.load(.acquire), reads), millisecondsPer(self.metrics.dma_stage_ns.load(.acquire), reads), millisecondsPer(self.metrics.tensor_init_ns.load(.acquire), reads), + millisecondsPer(self.metrics.ahead_init_ns.load(.acquire), 1), }); const submissions = self.metrics.dma_submissions.load(.acquire); load_log.debug("loader DMA: dma_submissions={d}, dma_submit_us_per_piece={d:.2}, dma_piece_latency_ms={d:.3}, pump_stops_empty={d}, pump_stops_full={d}", .{ @@ -609,10 +678,12 @@ pub const Loader = struct { /// One submission: its plans (one per source file, published as each file /// is planned), the destinations it writes and their parts, and every /// request and event context created while loading it, all in the batch's -/// arena until it retires. `remaining` counts completion units: one -/// per published job plus a publish sentinel held until the submission is -/// sealed. A job's unit is released exactly once, by whichever of these -/// happens: its request's last reference drops (final DMA callback or +/// arena until it retires. `remaining` counts completion units: one per +/// published job plus a publish sentinel held until the submission is +/// sealed, and one for the allocation task while it runs +/// (`Loader.startAllocations`), taken before the sentinel is dropped. A +/// job's unit is released exactly once, by whichever of these happens: its +/// request's last reference drops (final DMA callback, a host job's end, or /// abandonment), or `Scheduler.fail` retires it unclaimed. The batch is /// done when `remaining` reaches zero; the awaiting task then retires it, /// so releasing a unit is the last permitted access to the batch. @@ -679,13 +750,17 @@ pub const Batch = struct { /// come from several files, so several plans can write it: their pieces, /// and the zero fill, all count toward the same targets (`Target.total` is /// the placement's bytes, whichever plan submits them), and its device -/// state is created once, by the first worker that reads or fills for it. +/// state is created once, by the first worker that reads or fills for it +/// or, for a host-resident one, normally by the allocation task ahead of +/// them. const Destination = struct { /// The caller's shape, which the output buffer keeps. shape: Shape, /// Where `shape`, packed and placed (`Platform.placedShape`), lands on /// its devices: what PJRT allocates and the planner cuts. grid: ShardGrid, + /// Where its bytes go (`TensorStore.Memory`). + memory: Memory = .default, output: *Buffer, /// What fills it, in ascending, disjoint destination order /// (`backend.LoadSpec`): a slice of the batch's parts. Bytes no part @@ -705,15 +780,16 @@ const Destination = struct { state: LazyOnce(DestinationState) = .{}, /// How its parts are planned, a class at a time in this order - /// (`Planner.sortedPartOrder`): DMA jobs, or gathered whole. - const Delivery = enum { dma, gathered }; + /// (`Planner.sortedPartOrder`): DMA jobs, host jobs, or gathered whole. + const Delivery = enum { dma, host, gathered }; fn delivery(self: *const Destination) Delivery { - return if (self.gathered) .gathered else .dma; + if (self.gathered) return .gathered; + return if (self.memory.isHost()) .host else .dma; } - fn ensureState(self: *Destination, allocator: std.mem.Allocator, io: std.Io, platform: *const Platform) StateInitError!*DestinationState { - return self.state.ensure(io, DestinationState.init, .{ allocator, platform, self }); + fn ensureState(self: *Destination, direct: *Loader) StateInitError!*DestinationState { + return self.state.ensure(direct.io, DestinationState.init, .{ direct, self }); } fn deinit(self: *Destination, allocator: std.mem.Allocator, api: *const pjrt.Api) void { @@ -771,10 +847,10 @@ const SourceSlot = struct { }; /// One file's jobs in claim order with their transfer records and every -/// context the jobs can need, allocated by the planner: one request per job -/// and one event per planned DMA submission, handed out in submission -/// order. Contexts hold the plan's address, which the batch's arena keeps -/// until it retires. +/// context the jobs can need, allocated by the planner: one request per +/// job and one event per planned DMA submission, handed out in +/// submission order; a host job needs only its request. Contexts hold the +/// plan's address, which the batch's arena keeps until it retires. const Plan = struct { /// A claimable job: its slice of the plan's transfers and what its /// worker does for them. Its request slot is its index. @@ -782,13 +858,18 @@ const Plan = struct { /// Where the job reads first: its read's start, a landing job's /// first extent's; zero for a job that reads nothing. /// `Planner.preparePlan` sorts the other jobs by it and appends - /// the fill job after them. + /// the fill jobs after them. file_offset: u64, transfer_start: usize, transfer_len: usize, kind: Kind, - /// How the job's bytes reach their destinations. + /// How the job's bytes reach their destinations. A destination + /// in a host memory space (`Memory.isHost`) is written + /// by the worker that reads it: going through pinned blocks and + /// a host-to-host PJRT transfer copied every byte twice and held + /// the DMA pumps, which on four B70 made Llama-3.1-8B take 2.0 s + /// into pinned_host against 0.67 s into device memory. const Kind = union(enum) { /// Read into pinned blocks in file order, where its transfers /// address it, then copied to the devices by DMA. @@ -801,6 +882,19 @@ const Plan = struct { /// Reads nothing: every transfer goes from the loader's zero /// block (`Planner.JobCut.appendFillTransfers`). dma_fill, + /// Read straight into host-resident destinations: the read's + /// iovecs are the destination ranges, so the read is the only + /// copy. Needs a read without alignment padding whose + /// transfers address each of its bytes once, in order + /// (`Planner.JobCut.coversReadInOrder`). + host_direct: Read, + /// Read into the request's slot of blocks, then copied into + /// host-resident destinations by the worker: reads widened + /// for direct I/O, whose buffers must be aligned, and + /// overlapping parts. + host_bounce: Read, + /// Writes zeros in place, reading nothing. + host_fill, }; /// `len` bytes from the job's `file_offset`, of which the first @@ -824,12 +918,12 @@ const Plan = struct { }; /// The blocks of its credit's slot the job reads into or lands - /// in; none for a fill job. + /// in; none for a direct host job or a fill job. fn blockLen(self: Job, block_size: usize) usize { return switch (self.kind) { - .dma_read => |read| Planner.blockCount(read.len, block_size), + .dma_read, .host_bounce => |read| Planner.blockCount(read.len, block_size), .dma_land => |land| Planner.blockCount(land.len, block_size), - .dma_fill => 0, + .dma_fill, .host_direct, .host_fill => 0, }; } }; @@ -868,10 +962,12 @@ const Plan = struct { item_len: usize, }; - /// `len` bytes at `landing` of the job's blocks taken end to end, for - /// every device of `writer_mask` at `destination_offset` of its shard. - /// Every transfer stays inside one block. A fill one lands at 0 and - /// reads the zero block. + /// `len` bytes at `landing` of the job's blocks taken end to end, + /// for every device of `writer_mask` at `destination_offset` of its + /// shard. Every transfer stays inside one block, which the worker or + /// `Dma.enqueue` finds from `landing` (`blockPiece`); a direct host + /// job reads at the same landings without blocks. A fill one lands + /// at 0: a DMA one reads the zero block, a host one writes zeros. const Transfer = struct { destination: *Destination, landing: usize, @@ -1014,11 +1110,11 @@ const Planner = struct { /// Plans one file: `order` indexes its `parts` by delivery class, then /// offset and size (`sortedPartOrder`). `JobCut.cutRuns` cuts the DMA - /// parts at the file's alignment (`Reads.alignmentOf`), + /// and the host parts at the file's alignment (`Reads.alignmentOf`), /// `JobCut.appendGatherJobs` each gathered destination with the file of /// its first part; the jobs merge in file order, then - /// `JobCut.appendFillJob` appends the zero fill. The plan goes in - /// `arena` (`JobCut.finish`), the cut in `allocator`. + /// `JobCut.appendFillJob` appends each class's zero fill. The plan goes + /// in `arena` (`JobCut.finish`), the cut in `allocator`. fn preparePlan( allocator: std.mem.Allocator, arena: std.mem.Allocator, @@ -1030,6 +1126,7 @@ const Planner = struct { const source_slot = parts[order[0]].source_slot; for (order) |part_index| std.debug.assert(std.mem.eql(u8, parts[part_index].source.file_uri, parts[order[0]].source.file_uri)); const dma_end = deliveryEnd(parts, order, 0, .dma); + const host_end = deliveryEnd(parts, order, dma_end, .host); const alignment = try reads.alignmentOf(source_slot); var cut: JobCut = .{ .allocator = allocator, @@ -1045,8 +1142,9 @@ const Planner = struct { .gather_len = maximumJobLen(config.request_size, reads.alignment), }; defer cut.deinit(); - try cut.cutRuns(order[0..dma_end]); - for (order[dma_end..]) |part_index| { + try cut.cutRuns(order[0..dma_end], false); + try cut.cutRuns(order[dma_end..host_end], true); + for (order[host_end..]) |part_index| { const part = &parts[part_index]; std.debug.assert(part.destination.gathered); if (part == &part.destination.parts[0]) try cut.appendGatherJobs(part.destination); @@ -1065,8 +1163,9 @@ const Planner = struct { // The zero fill of each destination goes out once, with the plan // of its first part; it depends on no read, so one job after this - // file's reads carries all of it. - try cut.appendFillJob(order[0..dma_end]); + // file's reads carries all of it, one per delivery class. + try cut.appendFillJob(order[0..dma_end], false); + try cut.appendFillJob(order[dma_end..host_end], true); return cut.finish(arena); } @@ -1114,11 +1213,16 @@ const Planner = struct { /// The plan of the jobs cut, copied into `arena` with every context /// its jobs can need, allocated before publication makes it visible - /// to workers: a request per job and an event per writer of a - /// transfer. + /// to workers: a request per job and an event per writer of a DMA + /// job's transfer. fn finish(self: *JobCut, arena: std.mem.Allocator) !*Plan { const jobs = self.jobs.items; const transfers = self.transfers.items; + var dma_submissions: usize = 0; + for (jobs) |job| switch (job.kind) { + .dma_read, .dma_land, .dma_fill => dma_submissions += submissionCount(transfers[job.transfer_start..][0..job.transfer_len]), + .host_direct, .host_bounce, .host_fill => {}, + }; const requests = try arena.alloc(ReadRequest, jobs.len); @memset(requests, ReadRequest.idle); const plan = try arena.create(Plan); @@ -1130,7 +1234,7 @@ const Planner = struct { .items = try arena.dupe(Plan.Item, self.items.items), .landings = try arena.dupe(u64, self.landings.items), .requests = requests, - .events = try arena.alloc(Dma.EventContext, submissionCount(transfers)), + .events = try arena.alloc(Dma.EventContext, dma_submissions), .source_bytes = self.source_bytes, }; return plan; @@ -1149,16 +1253,18 @@ const Planner = struct { self.jobs.deinit(allocator); } - /// Cuts the runs of `order` (the DMA parts' indices in file order) - /// into jobs: each run of touching or overlapping ranges into the - /// minimum number of jobs, cut at part-safe boundaries where that - /// number allows. No job depends on another, since every DMA piece - /// is submitted as soon as its block is read. With an alignment - /// (`Reads.alignmentOf`), a job's read widens to aligned bounds - /// around its range (`maximumJobLen` leaves room for it), and the - /// transfers address the widened read. Each job's segments are - /// placed in its blocks by `placeDma`. - fn cutRuns(self: *JobCut, order: []const usize) !void { + /// Cuts the runs of `order` (part indices of one delivery class in + /// file order) into jobs: each run of touching or overlapping + /// ranges into the minimum number of jobs, cut at part-safe + /// boundaries where that number allows. No job depends on another, + /// since every DMA piece is submitted as soon as its block is read. + /// With an alignment (`Reads.alignmentOf`), a job's read widens to + /// aligned bounds around its range (`maximumJobLen` leaves room for + /// it), and the transfers address the widened read. A DMA job's + /// segments are placed in its blocks by `placeDma`. A host job is + /// `host_direct` only when its read is exact and its transfers + /// cover it in order (`coversReadInOrder`), otherwise `host_bounce`. + fn cutRuns(self: *JobCut, order: []const usize, host: bool) !void { const allocator = self.allocator; const parts = self.parts; const alignment = self.alignment; @@ -1255,7 +1361,13 @@ const Planner = struct { } std.debug.assert(self.segments.items.len != 0); const read: Plan.Job.Read = .{ .len = read_len, .minimum_len = @intCast(job_end - read_start) }; - const kind = try self.placeDma(read, read_start, job_start, job_len); + const kind: Plan.Job.Kind = if (host) host: { + for (self.segments.items) |segment| try self.appendRunTransfers(transfer_start, segment, read_start); + break :host if (alignment == 0 and coversReadInOrder(self.transfers.items[transfer_start..], read_len)) + .{ .host_direct = read } + else + .{ .host_bounce = read }; + } else try self.placeDma(read, read_start, job_start, job_len); try self.jobs.append(allocator, .{ .file_offset = read_start, .transfer_start = transfer_start, @@ -1607,10 +1719,11 @@ const Planner = struct { } /// The zero fill of the destinations whose first part is among - /// `order`, the file's DMA parts, as one job that reads one block, - /// the zero block. A gathered destination's jobs write its zeros + /// `order`, the file's parts of one delivery class, as one job. A DMA + /// fill job reads one block, the zero block; a host one writes zeros + /// in place. A gathered destination's jobs write its zeros /// themselves. - fn appendFillJob(self: *JobCut, order: []const usize) !void { + fn appendFillJob(self: *JobCut, order: []const usize, host: bool) !void { const fill_start = self.transfers.items.len; for (order) |part_index| { const part = &self.parts[part_index]; @@ -1624,14 +1737,14 @@ const Planner = struct { .file_offset = 0, .transfer_start = fill_start, .transfer_len = fill_transfers.len, - .kind = .dma_fill, + .kind = if (host) .host_fill else .dma_fill, }); } /// Appends the zero-fill transfers of `destination`: every logical /// range no part covers (the gaps between its parts and the tail), /// placed by its grid like read bytes, each piece at most one block - /// at landing 0, where the job reads the zero block. Pieces count + /// at landing 0, where a DMA job reads the zero block. Pieces count /// toward their targets' totals, so the fill completes each /// device's placement bytes alongside the parts. fn appendFillTransfers(self: *JobCut, transfer_start: usize, destination: *Destination) !void { @@ -1680,6 +1793,20 @@ const Planner = struct { } } } + + /// Whether a host job's transfers address its read from byte zero + /// to `read_len` in order, each source byte once: then the read + /// can scatter straight into the destinations. Overlapping or + /// duplicate parts address some source bytes twice and go through + /// blocks instead. + fn coversReadInOrder(transfers: []const Plan.Transfer, read_len: usize) bool { + var next: usize = 0; + for (transfers) |transfer| { + if (transfer.landing != next) return false; + next += transfer.len; + } + return next == read_len; + } }; fn blockCount(len: usize, block_size: usize) usize { return len / block_size + @intFromBool(len % block_size != 0); @@ -1937,21 +2064,24 @@ const ReadRequest = struct { fn run(self: *ReadRequest, scratch: *Scratch) !void { const loader = self.loader; + const io = loader.io; const block_size = loader.plan_config.block_size; const job = self.plan.jobs[self.index]; const transfers = self.plan.transfersOf(job); - // The first worker to touch a destination creates its state. + // The first worker to touch a destination creates its state; a + // host-resident one's is normally created ahead by the allocation + // task. var previous: ?*Destination = null; for (transfers) |transfer| { const destination = transfer.destination; if (destination == previous) continue; previous = destination; - const init_started = awakeNs(loader.io); - _ = destination.ensureState(loader.allocator, loader.io, loader.platform) catch |err| { + const init_started = awakeNs(io); + _ = destination.ensureState(loader) catch |err| { loader.recordError(err, "initialize tensor {s} from {s}, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); return err; }; - _ = loader.metrics.tensor_init_ns.fetchAdd(awakeNs(loader.io) -| init_started, .monotonic); + _ = loader.metrics.tensor_init_ns.fetchAdd(awakeNs(io) -| init_started, .monotonic); } const blocks = loader.credits.slotBlocks(self.slot)[0..job.blockLen(block_size)]; switch (job.kind) { @@ -1959,9 +2089,50 @@ const ReadRequest = struct { .dma_land => |land| try landJob(loader, self.plan, land, blocks, scratch), // Reads nothing: its transfers read the zero block. .dma_fill => return Dma.enqueue(self, loader.blocks[0..1]), + // A host job's worker writes its destinations itself, so the job + // ends with the worker's scheduling reference; its slot's blocks, + // if it reads through them, are done when this returns. + .host_direct => |read| { + var scatter: Scatter = .{ + .loader = loader, + .slot = self.plan.source_slot, + .iovecs = scratch.scatterIovecs(), + .file_offset = job.file_offset, + }; + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + // The planner checked that the transfers cover the read + // in order, each source byte once. + try scatter.add(tensor.hostLanding(transfer, @ctz(transfer.writer_mask))); + } + try scatter.flush(); + std.debug.assert(scatter.file_offset == job.file_offset + read.len); + for (transfers) |transfer| transfer.destination.state.readyValue().?.copyToReplicas(transfer); + }, + .host_bounce => |read| { + try self.readBlocks(scratch, blocks, read); + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + // The last block holds stale bytes past the read, which + // no bounds check would stop a copy from taking. + std.debug.assert(transfer.landing + transfer.len <= read.len); + const piece = blockPiece(blocks, block_size, transfer.landing, transfer.len); + std.debug.assert(piece.len == transfer.len); + var writers = transfer.writer_mask; + while (writers != 0) : (writers &= writers - 1) @memcpy(tensor.hostLanding(transfer, @ctz(writers)), piece); + } + }, + .host_fill => { + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + var writers = transfer.writer_mask; + while (writers != 0) : (writers &= writers - 1) @memset(tensor.hostLanding(transfer, @ctz(writers)), 0); + } + return; + }, } for (transfers) |transfer| transfer.destination.recordRead(transfer.len); - try Dma.enqueue(self, blocks); + if (job.kind == .dma_read or job.kind == .dma_land) try Dma.enqueue(self, blocks); } /// Reads the job's `read` into `blocks`, its slot's, in file order. @@ -2175,9 +2346,9 @@ const max_positional_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 /// A read scattered over memory ranges as they come, in file order and /// back to back from `file_offset` of `slot`'s file: a range that continues /// the previous one in memory merges with it, and one call reads at most -/// `iovecs.len` ranges. Fed the runs of a walk landing in `blocks` -/// (`place`), or memory ranges directly (`add`). Every byte read must -/// exist: a scattered read has no alignment padding. +/// `iovecs.len` ranges. Fed destination ranges directly (`add`), or the runs +/// of a walk landing in `blocks` (`place`). Every byte read must exist: a +/// scattered read has no alignment padding. const Scatter = struct { loader: *Loader, slot: *SourceSlot, @@ -2640,15 +2811,24 @@ const DestinationState = struct { } }; - /// The destination's transfer targets, one per device in canonical - /// order. - targets: []Target, + /// A DMA destination's transfer targets, one per device in canonical + /// order; empty for a host-resident one. + targets: []Target = &.{}, + /// A host-resident destination's shards in host memory, which workers + /// write in place, one per device in canonical order; empty for a DMA + /// one. + host: [][*]u8 = &.{}, - /// Creates the destination's device buffers and transfer managers, one - /// per device of its grid in canonical order, and writes the output - /// shell. Runs once per destination (`LazyOnce`), however many parts - /// feed it, on the worker that first reads or fills for it. - fn init(allocator: std.mem.Allocator, platform: *const Platform, destination: *const Destination) StateInitError!DestinationState { + /// Creates the destination's buffers and writes the output shell: for a + /// DMA destination, the device buffers and transfer managers, one per + /// device of its grid in canonical order; for a host-resident one, the + /// buffers workers write (`initHost`). Runs once per destination + /// (`LazyOnce`), however many parts feed it, on the allocation task or + /// the worker that first reads or fills for it. + fn init(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { + if (destination.memory.isHost()) return initHost(direct, destination); + const allocator = direct.allocator; + const platform = direct.platform; const grid = &destination.grid; const targets = try allocator.alloc(Target, grid.devices.len); errdefer allocator.free(targets); @@ -2685,20 +2865,91 @@ const DestinationState = struct { return .{ .targets = targets }; } + /// Creates a host-resident destination's buffers, one per device of its + /// grid in canonical order, each in that device's pinned host memory + /// space, and reads their host pointers for the workers. The buffers are + /// complete before any worker writes them, so they report ready from the + /// start; the front end hands them out only once the batch is done. + fn initHost(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { + const allocator = direct.allocator; + const platform = direct.platform; + const api = platform.pjrt_api; + const grid = &destination.grid; + const shards = try allocator.alloc([*]u8, grid.devices.len); + errdefer allocator.free(shards); + const dims = grid.local.dims(); + const element_type = pjrtx.bufferTypeFromDtype(grid.local.dtype()); + const layout = platform.defaultMemoryLayout(dims, grid.local.dtype()); + + var pjrt_buffers: Buffer.Shards = .empty; + errdefer for (pjrt_buffers.constSlice()) |buffer| buffer.deinit(api); + for (grid.devices, shards) |device, *shard| { + // The front end only submits host kinds when every device has + // a pinned host space (`backend.supportsMemory`). + const host = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; + const buffer = try createHostBuffer(direct.io, platform, dims, element_type, layout, host); + pjrt_buffers.appendAssumeCapacity(buffer); + shard.* = @ptrCast(try buffer.opaqueDeviceMemoryDataPointer(api)); + } + + destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); + return .{ .host = shards }; + } + fn deinit(self: *DestinationState, allocator: std.mem.Allocator, api: *const pjrt.Api) void { for (self.targets) |target| target.manager.deinit(api); allocator.free(self.targets); + allocator.free(self.host); + } + + /// Where a host transfer's bytes go on `writer`, one of its writers (a + /// canonical device index of `writer_mask`). + fn hostLanding(self: *const DestinationState, transfer: Plan.Transfer, writer: usize) []u8 { + std.debug.assert(writer < self.host.len and transfer.writer_mask >> @as(u6, @intCast(writer)) & 1 != 0); + return self.host[writer][transfer.destination_offset..][0..transfer.len]; + } + + /// Copies a host transfer's bytes from its first writer's shard, where a + /// direct read put them, to its other writers'. + fn copyToReplicas(self: *const DestinationState, transfer: Plan.Transfer) void { + const first = self.hostLanding(transfer, @ctz(transfer.writer_mask)); + var others = transfer.writer_mask & (transfer.writer_mask - 1); + while (others != 0) : (others &= others - 1) @memcpy(self.hostLanding(transfer, @ctz(others)), first); } }; +/// An uninitialized buffer in a host memory space, once its allocation +/// completed: its host pointer is valid only then. +fn createHostBuffer( + io: std.Io, + platform: *const Platform, + dims: []const i64, + element_type: pjrt.BufferType, + layout: pjrt.MemoryLayout, + memory: *const pjrt.Memory, +) StateInitError!*pjrt.Buffer { + const api = platform.pjrt_api; + const buffer = try platform.pjrt_client.createUninitializedBuffer(api, .{ + .dims = dims, + .element_type = element_type, + .layout = layout, + .dst = .{ .memory = memory }, + }); + errdefer buffer.deinit(api); + const ready = buffer.readyEvent(api); + defer ready.deinit(api); + try ready.await(api, io); + return buffer; +} + /// The lifecycle credits: one per request the pre-grown pinned set holds, /// each a fixed slot of `blocks_per_slot` blocks. A worker takes a credit /// before claiming a job, and the request gives it back with its last -/// reference, after its last DMA callback. Only the holder reads into and -/// transfers from its slot's blocks, so the credits are the one bound on -/// pinned memory in use. A block whose pieces are done stays with its -/// request until the request ends; nobody else could use it anyway, since -/// every credit owns a whole slot. +/// reference: after its last DMA callback, or at the end of a host job. Only +/// the holder reads into, lands in and transfers from its slot's blocks, so +/// the credits are the one bound on pinned memory in use. A block whose +/// pieces are done stays with its request until the request ends; nobody +/// else could use it anyway, since every credit owns a whole slot. const Credits = struct { /// Every slot's blocks, slot `s` at `blocks[s * blocks_per_slot ..]`; /// a tail shorter than a slot is left unused. @@ -2790,8 +3041,12 @@ const Metrics = struct { pump_stops_full: std.atomic.Value(u64) = .init(0), /// Inside `ensureState` for a claimed job's destinations: the first /// worker to touch a tensor creates its PJRT buffers and transfer - /// managers there, and the other workers of the same tensor wait for it. + /// managers there, and the other workers of the same tensor wait for it; + /// a host job waits there for the allocation task. tensor_init_ns: std.atomic.Value(u64) = .init(0), + /// The allocation task's time creating host-resident destinations ahead + /// of their reads, summed over batches. + ahead_init_ns: std.atomic.Value(u64) = .init(0), }; /// A value initialized at most once by whichever task touches it first, @@ -3090,6 +3345,64 @@ test "source planner coalesces exact adjacent and overlapping tensor ranges per try std.testing.expectEqual(@as(u64, 20), widened_plan.source_bytes); } +test "host-resident and DMA parts never share a job, and overlapping host parts read through blocks" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 2 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + // x, y and z end to end in one file. x lands in two host destinations, + // y in device memory between them, z in a third host destination. + const shape = Shape.init(.{ 8, 8 }, .u8); + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "file", .name = "x", .shape = shape, .offset = 0 }, + .{ .file_uri = "file", .name = "y", .shape = shape, .offset = 64 }, + .{ .file_uri = "file", .name = "z", .shape = shape, .offset = 128 }, + }; + var slot: SourceSlot = .{ .uri = "file" }; + var outputs: [4]Buffer = undefined; + var destinations: [4]Destination = undefined; + var parts = [_]Part{ + .{ .source = &sources[0], .source_slot = &slot, .destination = &destinations[0], .lattice = .{ .width = 64 } }, + .{ .source = &sources[1], .source_slot = &slot, .destination = &destinations[1], .lattice = .{ .width = 64 } }, + .{ .source = &sources[0], .source_slot = &slot, .destination = &destinations[2], .lattice = .{ .width = 64 } }, + .{ .source = &sources[2], .source_slot = &slot, .destination = &destinations[3], .lattice = .{ .width = 64 } }, + }; + const memories = [_]Memory{ .host_pinned, .default, .host_pinned, .host_pinned }; + for (&destinations, &outputs, &parts, memories) |*destination, *output, *part, memory| { + destination.* = .{ .shape = shape, .grid = .init(platform, shape), .memory = memory, .output = output, .parts = @as(*[1]Part, part), .unread_bytes = .init(64) }; + } + + const order = try Planner.sortedPartOrder(allocator, &parts); + defer allocator.free(order); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const plan = try prepareTestPlan(arena.allocator(), &parts, order, .{ + .block_size = 64, + .request_size = 1024, + }, 0); + // x's run reads its bytes once for two destinations, so through + // blocks; y goes out by DMA; z is read in place. File order. + const expected = [_]struct { offset: u64, kind: std.meta.Tag(Plan.Job.Kind) }{ + .{ .offset = 0, .kind = .host_bounce }, + .{ .offset = 64, .kind = .dma_read }, + .{ .offset = 128, .kind = .host_direct }, + }; + try std.testing.expectEqual(expected.len, plan.jobs.len); + var dma_writers: usize = 0; + for (plan.jobs, expected) |job, want| { + try std.testing.expectEqual(want.offset, job.file_offset); + try std.testing.expectEqual(want.kind, std.meta.activeTag(job.kind)); + for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| { + try std.testing.expectEqual(job.kind != .dma_read, transfer.destination.memory.isHost()); + if (job.kind == .dma_read) dma_writers += @popCount(transfer.writer_mask); + } + } + try std.testing.expectEqual(@as(usize, 2), plan.jobs[0].transfer_len); + try std.testing.expectEqual(@as(usize, 1), plan.jobs[0].blockLen(64)); + try std.testing.expectEqual(dma_writers, plan.events.len); +} + /// Plans a composite destination fed by three parts from two files, with a /// gap and a tail of zero fill, beside a plain one, then replays the plans on /// host memory: every device's placement bytes are written exactly once, @@ -3098,6 +3411,7 @@ const CompositeTest = struct { const Options = struct { block_size: usize, request_size: usize, + memory: Memory = .default, alignment: usize = 0, /// By default nothing is staged. staging_gain: usize = std.math.maxInt(usize), @@ -3105,9 +3419,10 @@ const CompositeTest = struct { cross_part_piece: bool, }; - /// A DMA job is staged where it saves `staging_gain` submissions; the - /// replay reads each job's blocks from the two files as its worker - /// would (`jobBlocks`). + /// With a host memory kind the destinations take host jobs: direct + /// without `alignment`, through its slot's blocks with it. A DMA job is + /// staged where it saves `staging_gain` submissions; the replay reads + /// its blocks from the two files as its worker would (`jobBlocks`). fn run(platform: *const Platform, shape: Shape, options: Options) !void { const allocator = std.testing.allocator; const io = std.testing.io; @@ -3149,9 +3464,10 @@ const CompositeTest = struct { }; const plain_shape = shape.setDim(0, 4); destinations = .{ - .{ .shape = shape, .grid = .init(platform, shape), .output = &outputs[0], .parts = parts[0..3], .unread_bytes = .init(144) }, - .{ .shape = plain_shape, .grid = .init(platform, plain_shape), .output = &outputs[1], .parts = parts[3..4], .unread_bytes = .init(64) }, + .{ .shape = shape, .grid = .init(platform, shape), .memory = options.memory, .output = &outputs[0], .parts = parts[0..3], .unread_bytes = .init(144) }, + .{ .shape = plain_shape, .grid = .init(platform, plain_shape), .memory = options.memory, .output = &outputs[1], .parts = parts[3..4], .unread_bytes = .init(64) }, }; + const host = destinations[0].memory.isHost(); const order = try Planner.sortedPartOrder(allocator, &parts); defer allocator.free(order); @@ -3171,17 +3487,26 @@ const CompositeTest = struct { } try std.testing.expectEqual(@as(usize, 2), plan_count); // The fill goes out once, with the plan of the destination's first - // part, as its last job; one event per writer of every transfer. - try std.testing.expectEqual(.dma_fill, std.meta.activeTag(plans[0].jobs[plans[0].jobs.len - 1].kind)); + // part, as its last job. + const fill_kind: std.meta.Tag(Plan.Job.Kind) = if (host) .host_fill else .dma_fill; + try std.testing.expectEqual(fill_kind, std.meta.activeTag(plans[0].jobs[plans[0].jobs.len - 1].kind)); + // Host jobs take no DMA event; DMA jobs one event per writer. for (plans[0..plan_count], [_]usize{ 1, 0 }) |plan, fill_jobs| { var writers: usize = 0; var fills: usize = 0; for (plan.jobs) |job| { - if (job.kind == .dma_fill) fills += 1; + const kind = std.meta.activeTag(job.kind); + if (kind == fill_kind) fills += 1; + if (host) { + const read_kind: std.meta.Tag(Plan.Job.Kind) = if (options.alignment == 0) .host_direct else .host_bounce; + try std.testing.expect(kind == .host_fill or kind == read_kind); + } else { + try std.testing.expect(kind == .dma_read or kind == .dma_land or kind == .dma_fill); + } for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| writers += @popCount(transfer.writer_mask); } try std.testing.expectEqual(fill_jobs, fills); - try std.testing.expectEqual(writers, plan.events.len); + try std.testing.expectEqual(if (host) 0 else writers, plan.events.len); } const zeros = try allocator.alloc(u8, 256); @@ -3196,17 +3521,27 @@ const CompositeTest = struct { } var cross_part_piece = false; for (plans[0..plan_count]) |plan| { + const file_bytes: []const u8 = if (plan.source_slot == &slots[0]) &first_bytes else &second_bytes; for (plan.jobs) |job| { - const fill = job.kind == .dma_fill; - const blocks = if (fill) &.{} else try jobBlocks(allocator, loader, plan, job); + const fill = job.kind == .dma_fill or job.kind == .host_fill; + const blocks = if (host or fill) &.{} else try jobBlocks(allocator, loader, plan, job); defer allocator.free(blocks); for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| { // Every transfer stays inside one block (`Plan.Transfer`). try std.testing.expect(transfer.landing % block_size + transfer.len <= block_size); - try std.testing.expect(transfer.landing < (if (fill) 1 else job.blockLen(block_size)) * block_size); + switch (job.kind) { + .host_direct, .host_bounce => |read| try std.testing.expect(transfer.landing + transfer.len <= read.len), + else => try std.testing.expect(transfer.landing < (if (fill) 1 else job.blockLen(block_size)) * block_size), + } const bytes = if (fill) fill: { try std.testing.expectEqual(@as(usize, 0), transfer.landing); break :fill zeros[0..transfer.len]; + } else if (host) read: { + // A host job's landings address its read, which + // starts at the job's file offset. + const file_offset = job.file_offset + transfer.landing; + if (plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; + break :read file_bytes[file_offset..][0..transfer.len]; } else read: { // a's last byte and b's first one in the same piece, // where a read's blocks hold the file in order. @@ -3244,7 +3579,7 @@ const CompositeTest = struct { return bytes; }, .dma_land => |land| land, - .dma_fill => unreachable, + .dma_fill, .host_direct, .host_bounce, .host_fill => unreachable, }; // A bounce buffer as large as any of the job's reads. var longest: usize = land.len; @@ -3367,6 +3702,12 @@ test "composite destinations place each part and zero-fill the rest of every dev try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .alignment = alignment, .staging_gain = 1, .cross_part_piece = scenario[2] }); try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 48, .alignment = alignment, .staging_gain = 1, .cross_part_piece = false }); } + // Host jobs, read in place or through blocks when the reads are + // widened, are cut by blocks the same way. + for ([_]usize{ 0, 8 }) |alignment| { + try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .memory = .host_pinned, .alignment = alignment, .cross_part_piece = scenario[2] }); + try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 32, .memory = .host_pinned, .alignment = alignment, .cross_part_piece = false }); + } } } } @@ -3994,9 +4335,9 @@ test "loader failures clean up before publication, after publication and during }; const parts = [_]LoadPart{ .whole(&sources[0]), .whole(&sources[1]) }; - // Device memory, the one kind this backend loads, goes through blocks - // and DMA. - for ([_]Memory{.default}) |memory| { + // Device memory goes through blocks and DMA; host memory is written in + // place by the workers. Both clean up the same way. + for ([_]Memory{ .default, .host_pinned }) |memory| { // Planning allocates from the scheduler's allocator, whose // allocations fail in turn: the first file's planning fails before // any plan is published, then the later file's after the first @@ -4463,3 +4804,88 @@ test "a strided part of a remote file is checked against its exact source range" try std.testing.expect(widened.read > widened.limit); } } + +/// A host-resident load on four devices: rows sharded; columns sharded, two +/// bytes per device per row, so one job has 8,192 transfers and scatters in +/// several reads; replicated; one source twice, replicated and row-sharded, +/// which overlaps; and a device-memory tensor in the same file. +const HostResidentTest = struct { + /// Each run writes its own file from `seed`: a buffer that lands on + /// memory an earlier run freed must not find the expected bytes there. + fn run(devices: *const testing.Devices, seed: u8, alignment: usize) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = devices.platform; + const mesh = try devices.mesh("tp"); + + var rows: [64 * 16]u8 = undefined; + var dev: [16 * 16]u8 = undefined; + var cols: [2048 * 8]u8 = undefined; + var rep: [96 * 4]u8 = undefined; + var twice: [32 * 4]u8 = undefined; + testing.pattern(&rows, seed); + testing.pattern(&dev, seed +% 1); + testing.pattern(&cols, seed +% 2); + testing.pattern(&rep, seed +% 3); + testing.pattern(&twice, seed +% 4); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + // The device tensor separates the host runs: an overlap sends its + // whole job through blocks, so the twice-bound source must not share + // a job with the others. + try testing.writeSafetensors(allocator, io, tmp.dir, "host.safetensors", ®istry, &.{ + .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 8 }, .bytes = &rows }, + .{ .name = "cols", .dtype = "U8", .dims = &.{ 2048, 8 }, .bytes = &cols }, + .{ .name = "rep", .dtype = "U8", .dims = &.{ 96, 4 }, .bytes = &rep }, + .{ .name = "dev", .dtype = "U16", .dims = &.{ 16, 8 }, .bytes = &dev }, + .{ .name = "twice", .dtype = "U8", .dims = &.{ 32, 4 }, .bytes = &twice }, + }); + + const names = [_][]const u8{ "rows", "dev", "cols", "rep", "twice", "twice" }; + const shapes = [_]Shape{ + Shape.init(.{ .row = 64, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }), + Shape.init(.{ .row = 16, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }), + Shape.init(.{ .row = 2048, .col = 8 }, .u8).withPartitioning(&mesh, .{ .col = .model }), + Shape.init(.{ .row = 96, .col = 4 }, .u8).withPartitioning(&mesh, .{}), + Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{}), + Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{ .row = .model }), + }; + const memories = [_]Memory{ .host_pinned, .default, .host_pinned, .host_pinned, .host_pinned, .host_pinned }; + const globals = [_][]const u8{ &rows, &dev, &cols, &rep, &twice, &twice }; + var parts: [names.len]LoadPart = undefined; + var specs: [names.len]LoadSpec = undefined; + var outputs: [names.len]Buffer = undefined; + for (&parts, &specs, &outputs, names, shapes, memories) |*part, *spec, *output, name, shape, memory| { + part.* = .whole(registry.tensors.getPtr(name).?); + spec.* = .{ .shape = shape, .memory = memory, .output = output, .parts = part[0..1] }; + } + + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + loader.reads.alignment = alignment; + const batch = try loader.submit(&specs, .none); + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + // Host shards in each device's pinned host memory. + for (&outputs, globals, memories) |*output, global, memory| try testing.expectShards(allocator, io, output, global, memory); + // Every credit, with its blocks, came back once the workers stopped + // (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + } +}; + +test "host-resident destinations are written in place, or through blocks when reads are widened or overlap" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 4); + defer devices.deinit(); + if (!backend.supportsMemory(devices.platform, .host_pinned)) return error.SkipZigTest; + // In place, then through blocks for every host job. + for ([_]usize{ 0, 64 }, [_]u8{ 1, 5 }) |alignment, seed| { + try HostResidentTest.run(&devices, seed, alignment); + } +} From 5e3b3cde53d9a5fbf0e3b170f04e85f65eef725a Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 16:08:49 +0100 Subject: [PATCH 086/103] zml/io: share one host allocation per replicated destination A replicated host_pinned destination kept a buffer per device, and its workers wrote every byte once per device. The devices holding one box now share one allocation: the first of them creates an owner buffer in its pinned host space, and every device's shard is a view of it. Each view holds a reference that its PJRT deletion callback drops, and the last drop destroys the owner, so the allocation outlives every reader whatever order the shards are freed in, the loader included. A host transfer is written once, through its first writer's shard. Writing through one view shows on every device, so a shard of such a destination must not be donated to an execution that writes it. The host-resident test checks that the replicated outputs share one pointer and one live allocation each, and that freeing the shards in reverse order releases them. Co-Authored-By: Claude Opus 5.5 --- zml/io/direct_loader.zig | 209 +++++++++++++++++++++++++++++++-------- 1 file changed, 170 insertions(+), 39 deletions(-) diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 9d1f3cd7..71082726 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -66,9 +66,10 @@ //! destinations when its read is exact and addresses each byte once, in //! order; a widened read, or parts that overlap, goes through its slot's //! blocks and is copied from there. Bytes no part covers are zeroed in -//! place. Each device of a replicated destination holds a copy of its own, -//! which the worker writes from the first one. Host and DMA parts never -//! share a job, and no strided part goes to host memory. +//! place. The devices holding one box share one allocation, through views +//! of it (`SharedHostAllocation`), so a replicated destination is written +//! once. Host and DMA parts never share a job, and no strided part goes to +//! host memory. //! //! Locks. Where several pumps are held, they are taken in device order. A //! pump's mutex may be held while a released request gives its credit back @@ -77,6 +78,7 @@ //! a time. const std = @import("std"); +const builtin = @import("builtin"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); @@ -2103,11 +2105,10 @@ const ReadRequest = struct { const tensor = transfer.destination.state.readyValue().?; // The planner checked that the transfers cover the read // in order, each source byte once. - try scatter.add(tensor.hostLanding(transfer, @ctz(transfer.writer_mask))); + try scatter.add(tensor.hostLanding(transfer)); } try scatter.flush(); std.debug.assert(scatter.file_offset == job.file_offset + read.len); - for (transfers) |transfer| transfer.destination.state.readyValue().?.copyToReplicas(transfer); }, .host_bounce => |read| { try self.readBlocks(scratch, blocks, read); @@ -2118,15 +2119,13 @@ const ReadRequest = struct { std.debug.assert(transfer.landing + transfer.len <= read.len); const piece = blockPiece(blocks, block_size, transfer.landing, transfer.len); std.debug.assert(piece.len == transfer.len); - var writers = transfer.writer_mask; - while (writers != 0) : (writers &= writers - 1) @memcpy(tensor.hostLanding(transfer, @ctz(writers)), piece); + @memcpy(tensor.hostLanding(transfer), piece); } }, .host_fill => { for (transfers) |transfer| { const tensor = transfer.destination.state.readyValue().?; - var writers = transfer.writer_mask; - while (writers != 0) : (writers &= writers - 1) @memset(tensor.hostLanding(transfer, @ctz(writers)), 0); + @memset(tensor.hostLanding(transfer), 0); } return; }, @@ -2816,7 +2815,7 @@ const DestinationState = struct { targets: []Target = &.{}, /// A host-resident destination's shards in host memory, which workers /// write in place, one per device in canonical order; empty for a DMA - /// one. + /// one. The devices of one box share theirs (`initHost`). host: [][*]u8 = &.{}, /// Creates the destination's buffers and writes the output shell: for a @@ -2867,29 +2866,62 @@ const DestinationState = struct { /// Creates a host-resident destination's buffers, one per device of its /// grid in canonical order, each in that device's pinned host memory - /// space, and reads their host pointers for the workers. The buffers are - /// complete before any worker writes them, so they report ready from the - /// start; the front end hands them out only once the batch is done. + /// space, and reads their host pointers for the workers. The devices + /// holding one box (`ShardGrid.writers`) share one allocation: every + /// device's shard is a view of it, and the allocation lives until the + /// last view is gone (`SharedHostAllocation`). A transfer is then + /// written once, through its first writer's shard (`hostLanding`). + /// Every direct target lets a host allocation of one device back a view + /// on another: CPU, CUDA and ROCm allocate portable pinned memory + /// (`Buffer.HostAccessible.init`), and on oneAPI four B70 gather from an + /// allocation made in the first one's host memory space, and read it + /// back. A target that could not would need a copy per device. + /// The buffers are complete before any worker writes them, so they + /// report ready from the start; the front end hands them out only once + /// the batch is done. fn initHost(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { const allocator = direct.allocator; const platform = direct.platform; const api = platform.pjrt_api; const grid = &destination.grid; - const shards = try allocator.alloc([*]u8, grid.devices.len); + const shard_count = grid.devices.len; + std.debug.assert(shard_count <= Platform.MAX_NUM_DEVICES); + const shards = try allocator.alloc([*]u8, shard_count); errdefer allocator.free(shards); const dims = grid.local.dims(); const element_type = pjrtx.bufferTypeFromDtype(grid.local.dtype()); const layout = platform.defaultMemoryLayout(dims, grid.local.dtype()); + // The writers of each device's box; the first of them creates the + // allocation the others view. + var box_writers: [Platform.MAX_NUM_DEVICES]u64 = undefined; + for (grid.writers[0..grid.box_count]) |writers| { + var mask = writers; + while (mask != 0) : (mask &= mask - 1) box_writers[@ctz(mask)] = writers; + } + + // Creation holds a reference to every shared allocation and its + // views one each. Declared before the buffers' cleanup, so a failure + // destroys the views first and this frees the owners. + var shared: [Platform.MAX_NUM_DEVICES]?*SharedHostAllocation = @splat(null); + defer for (shared[0..shard_count]) |allocation| if (allocation) |a| a.release(); var pjrt_buffers: Buffer.Shards = .empty; errdefer for (pjrt_buffers.constSlice()) |buffer| buffer.deinit(api); - for (grid.devices, shards) |device, *shard| { + for (grid.devices, shards, 0..) |device, *shard, index| { // The front end only submits host kinds when every device has // a pinned host space (`backend.supportsMemory`). const host = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; - const buffer = try createHostBuffer(direct.io, platform, dims, element_type, layout, host); - pjrt_buffers.appendAssumeCapacity(buffer); - shard.* = @ptrCast(try buffer.opaqueDeviceMemoryDataPointer(api)); + if (grid.replicas() > 1) { + const first: usize = @ctz(box_writers[index]); + if (first == index) shared[index] = try .create(direct.io, platform, dims, element_type, layout, host); + const allocation = shared[first].?; + pjrt_buffers.appendAssumeCapacity(try allocation.view(platform, dims, element_type, layout, host)); + shard.* = allocation.data; + } else { + const buffer = try createHostBuffer(direct.io, platform, dims, element_type, layout, host); + pjrt_buffers.appendAssumeCapacity(buffer); + shard.* = @ptrCast(try buffer.opaqueDeviceMemoryDataPointer(api)); + } } destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); @@ -2902,19 +2934,93 @@ const DestinationState = struct { allocator.free(self.host); } - /// Where a host transfer's bytes go on `writer`, one of its writers (a - /// canonical device index of `writer_mask`). - fn hostLanding(self: *const DestinationState, transfer: Plan.Transfer, writer: usize) []u8 { - std.debug.assert(writer < self.host.len and transfer.writer_mask >> @as(u6, @intCast(writer)) & 1 != 0); - return self.host[writer][transfer.destination_offset..][0..transfer.len]; + /// Where a host transfer's bytes go: the allocation its writers, the + /// devices of one box, share. Writing it once writes every one of them. + fn hostLanding(self: *const DestinationState, transfer: Plan.Transfer) []u8 { + const first: usize = @ctz(transfer.writer_mask); + std.debug.assert(first < self.host.len); + return self.host[first][transfer.destination_offset..][0..transfer.len]; + } +}; + +/// One host allocation that the devices of a box read through views of it, +/// in their pinned host memory space: the owner buffer is not a shard, +/// every view holds a reference that its deletion callback drops, and the +/// last drop destroys the owner. PJRT runs a view's callback once the +/// view's memory is out of use, after any execution still reading it, so +/// the allocation outlives its readers whatever order the shards are freed +/// in, including views of the shards (`Buffer.createView`) that their owner +/// outlives. On the process heap: the last view may go long after the +/// loader. +/// Writing through one view shows on every device, so a shard of a +/// replicated host-resident destination must not be donated to an +/// execution that writes it, as with `Buffer.HostAccessible.init(.undonatable)`. +const SharedHostAllocation = struct { + /// Allocations not released yet, counted in tests only. + var live: if (builtin.is_test) std.atomic.Value(usize) else void = if (builtin.is_test) .init(0) else {}; + + api: *const pjrt.Api, + owner: *pjrt.Buffer, + data: [*]u8, + /// One per live view, and one for `DestinationState.initHost` while it + /// creates them. + references: std.atomic.Value(usize) = .init(1), + + fn create( + io: std.Io, + platform: *const Platform, + dims: []const i64, + element_type: pjrt.BufferType, + layout: pjrt.MemoryLayout, + memory: *const pjrt.Memory, + ) StateInitError!*SharedHostAllocation { + const api = platform.pjrt_api; + const self = try std.heap.smp_allocator.create(SharedHostAllocation); + errdefer std.heap.smp_allocator.destroy(self); + const owner = try createHostBuffer(io, platform, dims, element_type, layout, memory); + errdefer owner.deinit(api); + self.* = .{ + .api = api, + .owner = owner, + .data = @ptrCast(try owner.opaqueDeviceMemoryDataPointer(api)), + }; + if (builtin.is_test) _ = live.fetchAdd(1, .monotonic); + return self; } - /// Copies a host transfer's bytes from its first writer's shard, where a - /// direct read put them, to its other writers'. - fn copyToReplicas(self: *const DestinationState, transfer: Plan.Transfer) void { - const first = self.hostLanding(transfer, @ctz(transfer.writer_mask)); - var others = transfer.writer_mask & (transfer.writer_mask - 1); - while (others != 0) : (others &= others - 1) @memcpy(self.hostLanding(transfer, @ctz(others)), first); + /// A shard in `memory`, holding a reference: the pinned host space of + /// its device. Every device can read it. + fn view( + self: *SharedHostAllocation, + platform: *const Platform, + dims: []const i64, + element_type: pjrt.BufferType, + layout: pjrt.MemoryLayout, + memory: *const pjrt.Memory, + ) StateInitError!*pjrt.Buffer { + _ = self.references.fetchAdd(1, .monotonic); + errdefer self.release(); + return platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ + .device_buffer_ptr = self.data, + .dims = dims, + .element_type = element_type, + .layout = layout, + .memory = memory, + .on_delete_callback = &onViewDeleted, + .on_delete_callback_arg = self, + }); + } + + fn onViewDeleted(_: ?*anyopaque, context: ?*anyopaque) callconv(.c) void { + const self: *SharedHostAllocation = @ptrCast(@alignCast(context.?)); + self.release(); + } + + fn release(self: *SharedHostAllocation) void { + if (self.references.fetchSub(1, .acq_rel) != 1) return; + self.owner.deinit(self.api); + std.heap.smp_allocator.destroy(self); + if (builtin.is_test) _ = live.fetchSub(1, .monotonic); } }; @@ -4865,19 +4971,44 @@ const HostResidentTest = struct { const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); defer loader.destroy(); loader.reads.alignment = alignment; - const batch = try loader.submit(&specs, .none); - try loader.awaitBatch(batch); - defer for (&outputs) |*output| output.deinit(); - // Host shards in each device's pinned host memory. - for (&outputs, globals, memories) |*output, global, memory| try testing.expectShards(allocator, io, output, global, memory); - // Every credit, with its blocks, came back once the workers stopped - // (`stopWorkers`). - loader.stopWorkers(); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + const live_before = SharedHostAllocation.live.load(.acquire); + { + const batch = try loader.submit(&specs, .none); + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + // Host shards in each device's pinned host memory. + for (&outputs, globals, memories) |*output, global, memory| try testing.expectShards(allocator, io, output, global, memory); + // Every credit, with its blocks, came back once the workers + // stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + + // The replicated host ones are one allocation. + for (outputs[3..5]) |output| { + const shards = output._shards.constSlice(); + const first = try shards[0].opaqueDeviceMemoryDataPointer(platform.pjrt_api); + for (shards[1..]) |shard| { + const pointer = try shard.opaqueDeviceMemoryDataPointer(platform.pjrt_api); + try std.testing.expect(pointer == first); + } + } + // The two replicated outputs. + try std.testing.expectEqual(live_before + 2, SharedHostAllocation.live.load(.acquire)); + // Shards may be freed in any order: free one replicated output's + // last first; its allocation goes with its last view. + std.mem.reverse(*pjrt.Buffer, outputs[3]._shards.slice()); + } + // PJRT may run the views' deletion callbacks on its own threads, + // just after the shards are freed. + var polls: usize = 0; + while (SharedHostAllocation.live.load(.acquire) != live_before) : (polls += 1) { + if (polls == 5000) return error.TestUnexpectedResult; + try io.sleep(.fromMilliseconds(1), .awake); + } } }; -test "host-resident destinations are written in place, or through blocks when reads are widened or overlap" { +test "host-resident destinations are written in place, through blocks when reads are widened or overlap, and replicas share one allocation" { const allocator = std.testing.allocator; const io = std.testing.io; var devices: testing.Devices = undefined; From f34318055a629ab00121b6bddf648c0eea72f781 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 16:12:39 +0100 Subject: [PATCH 087/103] zml/io: deliver host_mapped destinations as device-kind views A tensor bound to host_mapped memory now loads: its bytes stay in one pinned host allocation per box, and each device's shard is a view of it in the device's default memory space, so kernels read the host memory in place. Its shards are always views, replicated or not, and the allocation goes with the last of them through the views' PJRT deletion callbacks; the loader keeps no owner. supportsMemory accepts both host kinds where the platform loads directly and every device has a pinned host space, and the front end refuses a host kind elsewhere. The front end's tests load a host_pinned and a host_mapped binding next to a device one, or check the refusal where the platform has no pinned host space, and gather rows of a padded table delivered host_mapped, sharded and replicated, on every device while it is freed behind queued matrix products, checking that its allocations outlive the gather and go once it is done and, where the plugin counts allocations, that the table took no device memory. The host-resident, failure and memory-kind loops run host_mapped too. Co-Authored-By: Claude Opus 5.5 --- zml/io/backend.zig | 15 +-- zml/io/direct_loader.zig | 69 +++++++---- zml/io/loader.zig | 256 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 299 insertions(+), 41 deletions(-) diff --git a/zml/io/backend.zig b/zml/io/backend.zig index 50b6f390..19f8a52e 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -393,17 +393,12 @@ pub fn loadsDirect(platform: *const Platform) bool { }; } -/// Whether the loader can deliver into `kind` on `platform`. `host_pinned` -/// needs the direct pipeline and a pinned host memory space on every -/// device. CPU loads through the direct backend, so it accepts it (its -/// plugin resolves every kind to its one memory space). `host_mapped` -/// needs views of host memory in device memory, which no backend makes. +/// Whether the loader can deliver into `kind` on `platform`. The host +/// kinds need the direct pipeline and a pinned host memory space on every +/// device. CPU loads through the direct backend, so it accepts them (its +/// plugin resolves every kind to its one memory space). pub fn supportsMemory(platform: *const Platform, kind: Memory) bool { - switch (kind) { - .default => return true, - .host_pinned => {}, - .host_mapped => return false, - } + if (kind == .default) return true; if (!loadsDirect(platform)) return false; for (platform.devices) |*device| { if (device.memory(.host_pinned) == null) return false; diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 71082726..753bf8c2 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -68,8 +68,10 @@ //! blocks and is copied from there. Bytes no part covers are zeroed in //! place. The devices holding one box share one allocation, through views //! of it (`SharedHostAllocation`), so a replicated destination is written -//! once. Host and DMA parts never share a job, and no strided part goes to -//! host memory. +//! once. A `host_mapped` destination's shards are always such views, in +//! the devices' default memory space, so kernels read the pinned memory in +//! place. Host and DMA parts never share a job, and no strided part goes +//! to host memory. //! //! Locks. Where several pumps are held, they are taken in device order. A //! pump's mutex may be held while a released request gives its credit back @@ -2870,12 +2872,15 @@ const DestinationState = struct { /// holding one box (`ShardGrid.writers`) share one allocation: every /// device's shard is a view of it, and the allocation lives until the /// last view is gone (`SharedHostAllocation`). A transfer is then - /// written once, through its first writer's shard (`hostLanding`). - /// Every direct target lets a host allocation of one device back a view - /// on another: CPU, CUDA and ROCm allocate portable pinned memory - /// (`Buffer.HostAccessible.init`), and on oneAPI four B70 gather from an - /// allocation made in the first one's host memory space, and read it - /// back. A target that could not would need a copy per device. + /// written once, through its first writer's shard (`hostLanding`). A + /// `host_mapped` shard is always such a view, on the device's default + /// memory space, so kernels read the allocation in place. Every direct + /// target lets a host allocation of one device back a view on another, + /// for transfers and kernels alike: CPU, CUDA and ROCm allocate portable + /// pinned memory (`Buffer.HostAccessible.init`), and on oneAPI four B70 + /// gather from an allocation made in the first one's host memory space, + /// and read it back. A target that could not would need a copy per + /// device. /// The buffers are complete before any worker writes them, so they /// report ready from the start; the front end hands them out only once /// the batch is done. @@ -2911,11 +2916,12 @@ const DestinationState = struct { // The front end only submits host kinds when every device has // a pinned host space (`backend.supportsMemory`). const host = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; - if (grid.replicas() > 1) { + if (grid.replicas() > 1 or destination.memory == .host_mapped) { const first: usize = @ctz(box_writers[index]); if (first == index) shared[index] = try .create(direct.io, platform, dims, element_type, layout, host); const allocation = shared[first].?; - pjrt_buffers.appendAssumeCapacity(try allocation.view(platform, dims, element_type, layout, host)); + const space = platform.devices[device.id].memory(destination.memory.space()).?.pjrt_memory; + pjrt_buffers.appendAssumeCapacity(try allocation.view(platform, dims, element_type, layout, space)); shard.* = allocation.data; } else { const buffer = try createHostBuffer(direct.io, platform, dims, element_type, layout, host); @@ -2944,14 +2950,14 @@ const DestinationState = struct { }; /// One host allocation that the devices of a box read through views of it, -/// in their pinned host memory space: the owner buffer is not a shard, -/// every view holds a reference that its deletion callback drops, and the -/// last drop destroys the owner. PJRT runs a view's callback once the -/// view's memory is out of use, after any execution still reading it, so -/// the allocation outlives its readers whatever order the shards are freed -/// in, including views of the shards (`Buffer.createView`) that their owner -/// outlives. On the process heap: the last view may go long after the -/// loader. +/// in their pinned host memory space or, for `host_mapped`, their default +/// one: the owner buffer is not a shard, every view holds a reference that +/// its deletion callback drops, and the last drop destroys the owner. PJRT +/// runs a view's callback once the view's memory is out of use, after any +/// execution still reading it, so the allocation outlives its readers +/// whatever order the shards are freed in, including views of the shards +/// (`Buffer.createView`) that their owner outlives. On the process heap: +/// the last view may go long after the loader. /// Writing through one view shows on every device, so a shard of a /// replicated host-resident destination must not be donated to an /// execution that writes it, as with `Buffer.HostAccessible.init(.undonatable)`. @@ -2988,8 +2994,9 @@ const SharedHostAllocation = struct { return self; } - /// A shard in `memory`, holding a reference: the pinned host space of - /// its device. Every device can read it. + /// A shard in `memory`, holding a reference: the host space of its + /// device, or the default one, where kernels read the allocation in + /// place. Every device can read it. fn view( self: *SharedHostAllocation, platform: *const Platform, @@ -3024,6 +3031,13 @@ const SharedHostAllocation = struct { } }; +/// Shared host allocations not released yet, in tests: the front end's +/// tests count the ones their views keep. +pub fn liveHostAllocations() usize { + comptime std.debug.assert(builtin.is_test); + return SharedHostAllocation.live.load(.acquire); +} + /// An uninitialized buffer in a host memory space, once its allocation /// completed: its host pointer is valid only then. fn createHostBuffer( @@ -4442,8 +4456,9 @@ test "loader failures clean up before publication, after publication and during const parts = [_]LoadPart{ .whole(&sources[0]), .whole(&sources[1]) }; // Device memory goes through blocks and DMA; host memory is written in - // place by the workers. Both clean up the same way. - for ([_]Memory{ .default, .host_pinned }) |memory| { + // place by the workers, into views of it for `host_mapped`. All clean + // up the same way. + for ([_]Memory{ .default, .host_pinned, .host_mapped }) |memory| { // Planning allocates from the scheduler's allocator, whose // allocations fail in turn: the first file's planning fails before // any plan is published, then the later file's after the first @@ -4958,7 +4973,8 @@ const HostResidentTest = struct { Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{}), Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{ .row = .model }), }; - const memories = [_]Memory{ .host_pinned, .default, .host_pinned, .host_pinned, .host_pinned, .host_pinned }; + // `rows` as views of pinned memory in the devices' default space. + const memories = [_]Memory{ .host_mapped, .default, .host_pinned, .host_pinned, .host_pinned, .host_pinned }; const globals = [_][]const u8{ &rows, &dev, &cols, &rep, &twice, &twice }; var parts: [names.len]LoadPart = undefined; var specs: [names.len]LoadSpec = undefined; @@ -4976,7 +4992,8 @@ const HostResidentTest = struct { const batch = try loader.submit(&specs, .none); try loader.awaitBatch(batch); defer for (&outputs) |*output| output.deinit(); - // Host shards in each device's pinned host memory. + // Host shards in each device's host memory, or its default + // memory for views. for (&outputs, globals, memories) |*output, global, memory| try testing.expectShards(allocator, io, output, global, memory); // Every credit, with its blocks, came back once the workers // stopped (`stopWorkers`). @@ -4992,8 +5009,8 @@ const HostResidentTest = struct { try std.testing.expect(pointer == first); } } - // The two replicated outputs. - try std.testing.expectEqual(live_before + 2, SharedHostAllocation.live.load(.acquire)); + // The two replicated outputs, and one per device of `rows`. + try std.testing.expectEqual(live_before + 2 + outputs[0]._shards.len, SharedHostAllocation.live.load(.acquire)); // Shards may be freed in any order: free one replicated output's // last first; its allocation goes with its last view. std.mem.reverse(*pjrt.Buffer, outputs[3]._shards.slice()); diff --git a/zml/io/loader.zig b/zml/io/loader.zig index 71605eb0..b4c27987 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -41,6 +41,7 @@ const Exe = @import("../exe.zig").Exe; const mem = @import("../mem.zig"); const Bufferized = mem.Bufferized; const meta = @import("../meta.zig"); +const ops = @import("../ops.zig"); const platform_mod = @import("../platform.zig"); const Memory = TensorStore.Memory; const Platform = platform_mod.Platform; @@ -54,6 +55,7 @@ const Backend = backend.Backend; const buffered_loader = @import("buffered_loader.zig"); const LoadPart = backend.LoadPart; const LoadSpec = backend.LoadSpec; +const direct_loader = @import("direct_loader.zig"); const load_log = @import("log.zig").load; const TensorStore = @import("TensorStore.zig"); @@ -259,8 +261,8 @@ pub const Loader = struct { /// rounded up to whole blocks of the smallest it may pick, 2 MiB or a /// pinned `dma_block_size`), a sub-byte last dimension that does not /// pack into whole bytes, a placement its mesh does not divide or on - /// devices the platform lacks, and a memory kind the platform cannot - /// take are `error.InvalidBinding`. + /// devices the platform lacks, and a host kind the platform cannot take + /// are `error.InvalidBinding`. /// /// `buffers` is `mem.bufferize` of `model`, whose buffers are empty. A /// tree of another structure (other union arms, optionals or slice @@ -1161,6 +1163,42 @@ test "tensors of one program and equal args share one compile, and one group run try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); } +test "loader delivers host bindings into pinned host memory" { + for ([_]Memory{ .host_pinned, .host_mapped }) |kind| try expectHostDelivery(kind); +} + +fn expectHostDelivery(kind: Memory) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const platform = fixture.platform; + const pinned = fixture.store.view().createReplicatedTensor("second", null); + fixture.store.view().setMemory(pinned, kind); + try std.testing.expectEqual(kind, fixture.store.bindingOf(pinned.id).?.load.memory); + + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + const Model = struct { device: Tensor, host: Tensor }; + const model: Model = .{ .device = fixture.value, .host = pinned }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + + if (!backend.supportsMemory(platform, kind)) { + // Refused before anything is submitted. + try std.testing.expectError(error.InvalidBinding, loader.start(&model, &buffers, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + return; + } + try loader.start(&model, &buffers, &fixture.store, .none); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.device, &LoaderTestFixture.contents, .default); + // One shard per device, each in that device's pinned host memory space + // or, `host_mapped`, a view of pinned memory in its default space. + try testing.expectShards(allocator, io, &buffers.host, &LoaderTestFixture.second_contents, kind); +} + test "a program frees its donated and non-donated inputs once" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -1452,8 +1490,7 @@ test "a rejected binding or a program of the wrong output fails the loader" { const splits = platform.devices.len > 1 and 4 % platform.devices.len == 0; // No binding, an empty source or program input, a mesh of other - // devices, a host kind the platform cannot take, and a program of the - // wrong shape or placement. + // devices, and a program of the wrong shape or placement. const Case = struct { tensor: Tensor, at_wait: bool = false, skip: bool = false }; const cases = [_]Case{ .{ .tensor = Tensor.fromShape(fixture.value.shape()) }, @@ -1462,7 +1499,6 @@ test "a rejected binding or a program of the wrong output fails the loader" { .{ .tensor = foreign[0] }, .{ .tensor = foreign[1] }, .{ .tensor = fixture.computed("value", foreign[2], LoaderTestFixture.Identity{}) }, - .{ .tensor = pinned, .skip = backend.supportsMemory(platform, .host_pinned) }, .{ .tensor = fixture.computed("value", fixture.value, Rows{}), .at_wait = true }, .{ .tensor = fixture.computed("value", misplaced, LoaderTestFixture.Identity{}), .at_wait = true, .skip = !splits }, }; @@ -2097,6 +2133,216 @@ test "a program takes its inputs from sources, in order" { try testing.expectShards(allocator, io, &output, &.{ 252, 252, 0, 0 }, .default); } +test "a padded host-mapped table stays in host memory and serves device gathers in place" { + // The engram arrangement: one source of 264-byte rows, an odd row count + // padded to a multiple of 8, rows sharded over every device, delivered + // as device-kind views of pinned host memory. A program gathers rows + // from each device's local shard. + try expectHostTableGathers(.rows); +} + +test "a replicated host-mapped table is one allocation that every device reads in place" { + // The same table on every device: the direct loader makes it one + // allocation (`direct_loader.DestinationState.initHost`), each device's + // shard a view of it, and every device's kernels gather from it. + try expectHostTableGathers(.replicated); +} + +/// Loads a padded table as `host_mapped`, rows sharded over every device or +/// replicated on each, and gathers rows of each device's shard on that +/// device. The table is freed while the gather is queued behind matrix +/// products: its memory must outlive the read, and go once it is done. +fn expectHostTableGathers(placement: enum { rows, replicated }) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const rows: usize = 100_003; + const row_bytes: usize = 264; + const gathered: usize = 256; + + // The accelerator under test, or four CPU devices. + const shared = testing.env(); + const owned: ?*Platform = if (shared.target == .cpu) + try Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 4 } }) + else + null; + defer if (owned) |platform| platform.deinit(allocator, io); + const platform: *const Platform = if (owned) |p| p else shared; + const api = platform.pjrt_api; + const mesh: *const Sharding.Mesh = if (owned) |p| + try p.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) + else + shared.meshes.get("model").?; + if (!backend.supportsMemory(platform, .host_mapped)) return error.SkipZigTest; + const devices = mesh.devicesInCanonicalOrder(); + const shard_count = devices.len; + + const padded_rows = std.mem.alignForward(usize, rows, 8); + const shard_rows = if (placement == .rows) padded_rows / shard_count else padded_rows; + if (placement == .rows) try std.testing.expectEqual(padded_rows, shard_rows * shard_count); + const padded = try allocator.alloc(u8, padded_rows * row_bytes); + defer allocator.free(padded); + @memset(padded, 0); + for (0..rows) |row| { + const bytes = padded[row * row_bytes ..][0..row_bytes]; + // The row's number first, so a misplaced row cannot pass for another. + std.mem.writeInt(u32, bytes[0..4], @intCast(row + 1), .little); + for (bytes[4..], 4..) |*byte, column| byte.* = @as(u8, @truncate(row *% 131 +% column *% 7)) | 1; + } + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "table.safetensors", ®istry, &.{ + .{ .name = "table", .dtype = "U8", .dims = &.{ @intCast(rows), @intCast(row_bytes) }, .bytes = padded[0 .. rows * row_bytes] }, + }); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{mesh} }); + defer store.deinit(); + const source = store.view().source("table", .{ .row, .channel }) + .pad(0, .{ .row = TensorStore.Source.Pad{ .high = @intCast(padded_rows - rows) } }); + const table = switch (placement) { + .rows => source.withPartitioning(.model, .{ .row = .model }).bind(), + .replicated => source.withPartitioning(.model, .{}).bind(), + }; + store.view().setMemory(table, .host_mapped); + try std.testing.expectEqual(@as(i64, @intCast(padded_rows)), table.dim(.row)); + + const live_before = direct_loader.liveHostAllocations(); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 4 }); + defer loader.deinit(); + // The device allocators' counts, once the set-up's calibration has + // allocated on the devices. Bytes in use would not do: CUDA's counts + // what its pool holds, which freed memory leaves room in. CPU's plugin + // counts nothing. + try loader.awaitSetup(); + var allocations: [Platform.MAX_NUM_DEVICES]?u64 = undefined; + for (devices) |device| allocations[device.id] = platform.devices[device.id].memoryStats().num_allocs; + var device_table: Buffer = undefined; + try loader.start(&table, &device_table, &store, .none); + try loader.wait(); + var freed = false; + defer if (!freed) device_table.deinit(); + // The table took no device memory: nothing was allocated on a device. + for (devices) |device| if (allocations[device.id]) |before| { + try std.testing.expectEqual(before, platform.devices[device.id].memoryStats().num_allocs.?); + }; + try std.testing.expectEqual(padded.len, loader.stats.bytes_loaded); + try testing.expectShards(allocator, io, &device_table, padded, .host_mapped); + // A row shard is one allocation per device, a replicated table one. + try std.testing.expectEqual(live_before + if (placement == .rows) shard_count else 1, direct_loader.liveHostAllocations()); + + const first_pointer = try device_table._shards.get(0).opaqueDeviceMemoryDataPointer(api); + for (devices, device_table._shards.constSlice()) |device, shard| { + // Replicas share one allocation; row shards never do. + const pointer = try shard.opaqueDeviceMemoryDataPointer(api); + const shares = placement == .replicated and shard_count > 1; + if (device.id != devices[0].id) try std.testing.expectEqual(shares, pointer == first_pointer); + } + + // Each device gathers `gathered` rows of its own shard, the padding + // rows of the last one included (of every replica, replicated), once + // a chain of matrix products is done: the indices depend on them. + const Gather = struct { + fn forward(table_: Tensor, indices: Tensor, ballast: Tensor) Tensor { + var product = ballast; + for (0..8) |_| product = product.dot(product.withTags(.{ .k, .j }), .k).withTags(.{ .i, .k }); + // Zero, as every product keeps the ballast's positive entries. + const offset: Tensor = .select( + product.sum(.i).sum(.k).reshape(Shape.init(.{}, .f32)).cmp(.LT, .scalar(0, .f32)), + .scalar(1, .u32), + .scalar(0, .u32), + ); + const output: Shape = Shape.init(.{ .shard = indices.dim(.shard), .bytes = indices.dim(.k) * table_.dim(.channel) }, .u8) + .withPartitioning(table_.shape()._sharding.mesh.?, .{ .shard = .model }); + return ops.manualComputation((struct { + fn body(inputs: struct { table: Tensor, indices: Tensor, offset: Tensor }, local: Shape) Tensor { + const shifted = inputs.indices.add(inputs.offset.broad(inputs.indices.shape())); + return inputs.table.gather(.{ .row = shifted }, .{}).reshape(local); + } + }).body, .{ .table = table_, .indices = indices, .offset = offset }, output); + } + }; + const indices_shape = Shape.init(.{ .shard = shard_count, .k = gathered }, .u32).withPartitioning(mesh, .{ .shard = .model }); + const indices = try allocator.alloc(u32, shard_count * gathered); + defer allocator.free(indices); + for (0..shard_count) |shard| { + const local = indices[shard * gathered ..][0..gathered]; + for (local, 0..) |*index, k| index.* = @intCast((k * 7919 + shard * 104_729) % shard_rows); + // The first and last rows of every shard; on the last device these + // are padding. + local[0] = 0; + local[1] = @intCast(shard_rows - 1); + local[2] = @intCast(shard_rows - 2); + } + var indices_buffer: Buffer = try .from(io, platform, indices_shape, std.mem.sliceAsBytes(indices), .{}); + defer indices_buffer.deinit(); + // Milliseconds of products on an accelerator; a token amount on CPU. + const ballast_dim: usize = if (platform.target == .cpu) 32 else 4096; + const ballast_shape = Shape.init(.{ .i = ballast_dim, .k = ballast_dim }, .f32).withPartitioning(mesh, .{}); + const ballast = try allocator.alloc(f32, ballast_dim * ballast_dim); + defer allocator.free(ballast); + @memset(ballast, 1.0 / @as(f32, @floatFromInt(ballast_dim))); + var ballast_buffer: Buffer = try .from(io, platform, ballast_shape, std.mem.sliceAsBytes(ballast), .{}); + defer ballast_buffer.deinit(); + + const exe = try platform.compileFn(allocator, io, Gather.forward, .{ Tensor.fromShape(table.shape()), Tensor.fromShape(indices_shape), Tensor.fromShape(ballast_shape) }, .{ .meshes = &.{mesh} }); + defer exe.deinit(); + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + args.set(.{ device_table, indices_buffer, ballast_buffer }); + try exe.call(io, args, &results, .{}); + var output = results.get(Buffer); + defer output.deinit(); + + // The table goes while the gather may still have to read it. Pinned + // buffers of its size and another pattern would take its memory had it + // gone early, and the gather would return their bytes. + device_table.deinit(); + freed = true; + const decoy_bytes = try allocator.alloc(u8, padded.len); + defer allocator.free(decoy_bytes); + @memset(decoy_bytes, 0xee); + var decoy: Buffer = try .from(io, platform, table.shape(), decoy_bytes, .{ .memory = .host_pinned }); + defer decoy.deinit(); + + const expected = try allocator.alloc(u8, shard_count * gathered * row_bytes); + defer allocator.free(expected); + for (0..shard_count) |shard| { + for (0..gathered) |k| { + const first_row = if (placement == .rows) shard * shard_rows else 0; + const row = first_row + indices[shard * gathered + k]; + @memcpy(expected[(shard * gathered + k) * row_bytes ..][0..row_bytes], padded[row * row_bytes ..][0..row_bytes]); + } + } + try testing.expectShards(allocator, io, &output, expected, .default); + + // The allocations go with their last views, once nothing reads them; + // PJRT may drop the views on its own threads just after the output is + // ready. + var polls: usize = 0; + while (direct_loader.liveHostAllocations() != live_before) : (polls += 1) { + if (polls == 5000) return error.TestUnexpectedResult; + try io.sleep(.fromMilliseconds(1), .awake); + } + + // The control: the same table in device memory is allocated on every + // device, so the check after the load above can fail. + const device_copy = switch (placement) { + .rows => source.withPartitioning(.model, .{ .row = .model }).bind(), + .replicated => source.withPartitioning(.model, .{}).bind(), + }; + for (devices) |device| allocations[device.id] = platform.devices[device.id].memoryStats().num_allocs; + var device_buffer: Buffer = undefined; + try loader.start(&device_copy, &device_buffer, &store, .none); + try loader.wait(); + defer device_buffer.deinit(); + for (devices) |device| if (allocations[device.id]) |before| { + try std.testing.expect(platform.devices[device.id].memoryStats().num_allocs.? > before); + }; +} + /// A loader on the buffered backend, whichever backend `platform` loads /// through (`backend.loadsDirect`), its set-up done in place: CPU loads /// through the direct backend, and this keeps the front end over the From b1d0abc96d94572d69d24e7a36ba400a017b75c7 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 16:21:41 +0100 Subject: [PATCH 088/103] zml/io: admit program groups by measured device room wait published a program group only once no other one was pending, so one group was in flight at a time whatever the devices held. It now admits a group as soon as it fits the room every device reports: the allocator's limit, minus the bytes in use, minus the bytes the direct backend was submitted but has not allocated yet, minus a 64 MiB reserve. A group costs its inputs' placements, its outputs' and the largest temporaries of its programs, from the executables' compiled memory stats (zero, logged once at debug, where the plugin gives none), beside the outputs and temporaries of the groups still pending. Pending submissions retire, oldest first, until the group fits; once no program group is pending, the room is read again after the retired groups' memory is given back, and the group goes even if it is still short, since the room is an estimate and can read low. A group that does not fit a device fails with the device's error, which wait returns; a smaller program_group_bytes makes smaller groups. Freed device memory stays counted in bytes_in_use until it is given back (Platform.releaseFreedDeviceMemory): XLA's VMM allocator, the default for CUDA and ROCm platforms, counts a freed allocation until it is reused or reclaimed. start gives back what was freed before the load, the compiles' scratch and the set-up's calibration buffers, so that neither the room nor the load's peak counts it. Admission gives back the retired groups' inputs only for a group that would otherwise go beyond the room, so the allocator still reuses them for the next group of the same shapes in the common case. Through llmd on two GB300, the one DeepSeek-V4.1-Flash group per load that goes beyond the room without the drain (3.4 to 4.3 GiB of room for 4.7 GiB) fits in 15.5 to 22.6 GiB, and MiMo-V2.6-Flash's peak device memory in use is 178-184 GB per device against 184-188 GB without these drains, for at most 49 ms per call on two devices. The direct backend counts each sealed batch's device bytes, a box per device of every destination outside host memory, and the bytes it allocates for outputs; their difference, saturating, is what it still owes the devices. Admission measures only when the backend is direct and every device reports a limit at the first start, so CPU and the buffered backend keep one group in flight. A debug line for each admission that retired or went beyond the room names the device the fit is decided on, and one at deinit reports the retires, the groups admitted beyond the room and the least room seen. Tests cover the room and fit arithmetic and the device the fit is decided on, check that the direct backend allocated exactly what was sealed for bulk, program and composite loads, and that host bindings count on neither side. Co-Authored-By: Claude Opus 5.5 --- zml/io/backend.zig | 4 + zml/io/direct_loader.zig | 35 +++- zml/io/loader.zig | 401 +++++++++++++++++++++++++++++++++++---- 3 files changed, 406 insertions(+), 34 deletions(-) diff --git a/zml/io/backend.zig b/zml/io/backend.zig index 19f8a52e..dceaeab5 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -406,6 +406,10 @@ pub fn supportsMemory(platform: *const Platform, kind: Memory) bool { return true; } +/// Bytes per device, by `platform.devices` index: `Platform.init` refuses +/// more devices than it holds. +pub const DeviceBytes = [Platform.MAX_NUM_DEVICES]u64; + /// The transfer implementation selected for the platform (`loadsDirect`). pub const Backend = union(enum) { direct: *direct_loader.Loader, diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 753bf8c2..5938fa86 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -155,6 +155,13 @@ pub const Loader = struct { /// read) and stays open until `destroy`. Keyed by the slot's own copy of /// the path: the registry that named it may go once the batch retired. source_slots: std.StringHashMapUnmanaged(*SourceSlot) = .empty, + /// Device bytes of the outputs of the sealed batches, per + /// `platform.devices` index, cumulative, as `DestinationState.init` + /// allocates them; counted by the submitting task. + sealed_bytes: backend.DeviceBytes = @splat(0), + /// Device bytes allocated for outputs so far, per `platform.devices` + /// index, cumulative. Freeing an output does not decrease them. + allocated_bytes: [Platform.MAX_NUM_DEVICES]std.atomic.Value(u64) = @splat(.init(0)), /// What the debug lines time from (`logBatch`, `logSummary`). created_at: std.Io.Timestamp, /// Submissions so far; the next batch's sequence number. @@ -310,6 +317,7 @@ pub const Loader = struct { // Stamped before the sentinel goes, so before the batch can complete. batch.sealed_at = .now(self.io, .awake); self.batch_count += 1; + self.countSealed(batch); self.startAllocations(batch); // Every plan is visible: drop the publish sentinel. A batch without // jobs completes right here. @@ -445,6 +453,25 @@ pub const Loader = struct { self.allocator.destroy(self); } + /// Output bytes of the sealed batches not allocated yet, per + /// `platform.devices` index. Workers allocate as they read, possibly + /// before the batch is sealed, so the difference saturates. + pub fn unallocatedBytes(self: *const Loader) backend.DeviceBytes { + var unallocated = self.sealed_bytes; + for (&unallocated, &self.allocated_bytes) |*bytes, *allocated| bytes.* -|= allocated.load(.acquire); + return unallocated; + } + + /// Counts the device bytes of the sealed `batch`: a box of each + /// destination outside host memory on each of its devices, as + /// `DestinationState.init` allocates them. + fn countSealed(self: *Loader, batch: *const Batch) void { + for (batch.destinations) |*destination| { + if (destination.memory.isHost()) continue; + for (destination.grid.devices) |device| self.sealed_bytes[device.id] += destination.grid.box_bytes; + } + } + fn checkOpen(self: *Loader) !void { if (self.first_error.get()) |err| return err; } @@ -2825,7 +2852,10 @@ const DestinationState = struct { /// device of its grid in canonical order; for a host-resident one, the /// buffers workers write (`initHost`). Runs once per destination /// (`LazyOnce`), however many parts feed it, on the allocation task or - /// the worker that first reads or fills for it. + /// the worker that first reads or fills for it, which also counts the + /// allocation once. A host destination (`Memory.isHost`) is created in + /// each device's pinned host memory space and takes no device memory, + /// so it is not counted in `allocated_bytes`. fn init(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { if (destination.memory.isHost()) return initHost(direct, destination); const allocator = direct.allocator; @@ -2861,6 +2891,9 @@ const DestinationState = struct { initialized += 1; pjrt_buffers.appendAssumeCapacity(pjrt_buffer); } + for (targets) |target| { + _ = direct.allocated_bytes[target.device_index].fetchAdd(target.total, .monotonic); + } destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); return .{ .targets = targets }; diff --git a/zml/io/loader.zig b/zml/io/loader.zig index b4c27987..52172959 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -1,11 +1,11 @@ //! Checkpoint loading. `Loader.start` takes a whole model: it starts //! compiling the programs that compute some of its tensors //! (`Source.bindProgram`) and publishes every tensor the loader fills -//! itself. `wait` then publishes the programs' inputs in groups, runs each -//! group's programs on the caller's task once its inputs landed, and -//! retires everything. Backends only read sources into buffers; lookup, -//! compilation, admission and execution stay in this shared front end. The -//! contract is in the pub doc comments below. +//! itself. `wait` then publishes the programs' inputs in groups the devices +//! have room for, runs each group's programs on the caller's task once its +//! inputs landed, and retires everything. Backends only read sources into +//! buffers; lookup, compilation, admission and execution stay in this +//! shared front end. The contract is in the pub doc comments below. //! //! From bindings to specs. `start` checks every tensor and program input //! once, before anything is published (`prepareSpec`, `checkPlacement`). @@ -25,14 +25,30 @@ //! do). //! //! Order and admission. The tensors the loader fills go out first, as one -//! submission. A group holds the tensors of consecutive programs in model -//! order, up to `program_group_bytes` of outputs, so the inputs of one -//! layer's experts, adjacent in the checkpoint, are read together. Before -//! publishing a group, `wait` waits for its programs to compile, then -//! retires the oldest pending submissions until no program group is -//! pending: one program group is in flight at a time. +//! submission, so every program group is admitted with them counted. A +//! group holds the tensors of consecutive programs in model order, up to +//! `program_group_bytes` of outputs, so the inputs of one layer's experts, +//! adjacent in the checkpoint, are read together. Before publishing a +//! group, `wait` waits for its programs to compile, then retires the oldest +//! pending submissions until the group fits the room every device reports +//! (`admission`): its limit, minus what is in use, minus the bytes the +//! direct backend published but has not allocated yet, minus a 64 MiB +//! reserve. A device's allocator may count freed memory as in use until it +//! is given back (`Platform.releaseFreedDeviceMemory`), so `start` gives +//! back what was freed before the load. A group costs its inputs, its +//! outputs and the largest of its programs' temporaries per device, since +//! they run one after the other. Once no program group is pending, retiring +//! frees nothing more: the room is read again after the retired groups' +//! memory is given back, and the group goes even if it is still short, +//! since the room is an estimate and can read low. A group that does not +//! fit a device fails its allocation or its execution with the device's +//! error, which `wait` returns; a smaller `Options.program_group_bytes` +//! makes smaller groups. Where a device reports no limit (CPU) or the +//! backend does not count its allocations (buffered), nothing is measured +//! and one program group is in flight at a time. const std = @import("std"); +const pjrt = @import("pjrt"); const VFS = @import("vfs"); const Buffer = @import("../buffer.zig").Buffer; @@ -55,8 +71,10 @@ const Backend = backend.Backend; const buffered_loader = @import("buffered_loader.zig"); const LoadPart = backend.LoadPart; const LoadSpec = backend.LoadSpec; +const DeviceBytes = backend.DeviceBytes; const direct_loader = @import("direct_loader.zig"); const load_log = @import("log.zig").load; +const ShardGrid = @import("ShardGrid.zig"); const TensorStore = @import("TensorStore.zig"); const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; @@ -119,6 +137,10 @@ pub const Loader = struct { pending: std.Deque(PendingSubmission) = .empty, /// The first error; every later call returns it. failure: ?anyerror = null, + /// The direct backend when admission can measure the room: it counts + /// its allocations and every device reported an allocator limit at the + /// first `start`. Otherwise one program group is in flight at a time. + measured: ?*direct_loader.Loader = null, stats: Stats = .{}, pub const Options = struct { @@ -198,13 +220,15 @@ pub const Loader = struct { return Backend.init(allocator, io, platform, opts); } - /// Waits for `init`'s task the first time: its backend, or its error, - /// for the caller to fail the loader with. + /// Waits for `init`'s task the first time: its backend, then whether + /// admission can measure the devices; or its error, for the caller to + /// fail the loader with. fn awaitSetup(self: *Loader) !void { const setup = if (self.setup) |*task| task else return; const result = setup.await(self.io); self.setup = null; self.backend = try result; + self.measured = self.probeMemory(); } /// Gives up on everything started: drops the queued groups, aborts the @@ -226,6 +250,7 @@ pub const Loader = struct { /// the caller's. Panics on loads neither waited for nor canceled. pub fn deinit(self: *Loader) void { if (self.loadsStarted()) std.debug.panic("Loader.deinit with loads started, whose store and buffers may be freed already: add `errdefer loader.cancel();` after `start`", .{}); + self.stats.logAdmission(self.measured != null); self.queued.deinit(self.allocator); self.pending.deinit(self.allocator); self.programs.deinit(self.allocator); @@ -241,6 +266,21 @@ pub const Loader = struct { return self.queued.len != 0 or self.pending.len != 0 or self.programs.items.len != 0; } + /// The direct backend when every device reports an allocator limit + /// (`measured`). Decided once: the CPU plugin logs an unimplemented + /// warning on every stats call, so admission never asks a device that + /// answered nothing. + fn probeMemory(self: *const Loader) ?*direct_loader.Loader { + const direct = switch (self.backend.?) { + .direct => |direct| direct, + .buffered => return null, + }; + for (self.platform.devices) |device| { + if (device.memoryStats().bytes_limit == null) return null; + } + return direct; + } + /// Starts loading every tensor of `model` into its buffer in `buffers`, /// as `store` binds it, and returns; `wait` returns once it is done. /// Each tensor is placed by its shape's sharding (replicated without a @@ -277,7 +317,8 @@ pub const Loader = struct { /// input; the backend completes them. /// Several loads may be started before one `wait`, from one store or /// several. The first `start` waits for `init`'s task, and fails with - /// its error. + /// its error. Each `start` then gives the device memory freed before it + /// back to the devices (`Platform.releaseFreedDeviceMemory`). pub fn start( self: *Loader, model: anytype, @@ -300,7 +341,12 @@ pub const Loader = struct { progress: std.Progress.Node, ) !void { try self.awaitSetup(); - // The load's time starts once the set-up is done. + // Give back what was freed before this load (compile scratch, the + // set-up's calibration buffers), so that admission's room does not + // count it and the load's peak does not include it. + self.platform.releaseFreedDeviceMemory(); + // The load's time starts once the set-up is done and the freed + // memory given back. self.stats.begin(self.io); var prepared = try prepareModel(self.allocator, self.platform, self.backend.?, store, model, buffers); errdefer prepared.arena.deinit(); @@ -313,12 +359,12 @@ pub const Loader = struct { prepared.arena.deinit(); return; } - try self.submit(prepared.specs, prepared.arena, &.{}, progress); + try self.submit(prepared.specs, prepared.arena, &.{}, @splat(0), progress); } /// Publishes the queued program groups, oldest first, each once its - /// programs compiled and no other program group is pending (`admit`), - /// and retires every submission in publish order, running a group's + /// programs compiled and the devices have room for it (`admit`), and + /// retires every submission in publish order, running a group's /// programs on this task once its inputs landed. Returns once every /// started load is in its buffers, with the freed device memory, its /// programs' inputs and temporaries among it, given back to the devices @@ -327,7 +373,8 @@ pub const Loader = struct { /// what the programs computed and how long publishing waited for their /// compiles. A program that fails to compile or run fails it; one that /// does not return its tensor's shape, placed alike, is - /// `InvalidBinding`. Idempotent: with nothing started it returns at + /// `InvalidBinding`; a group that does not fit a device fails it with + /// the device's error. Idempotent: with nothing started it returns at /// once and logs nothing. pub fn wait(self: *Loader) !void { if (self.failure) |err| return err; @@ -427,25 +474,110 @@ pub const Loader = struct { self.programs.clearRetainingCapacity(); } - /// Publishes `group` once its programs compiled and no other program - /// group is pending: its calls and its specs' arena move to the pending + /// Publishes `group` once its programs compiled and the devices have + /// room for it: its calls and its specs' arena move to the pending /// submission, or are freed when this fails. fn publish(self: *Loader, group: Group) !void { errdefer group.deinit(self.allocator); const started: std.Io.Timestamp = .now(self.io, .awake); for (group.calls) |call| try call.program.await(self.io, self.platform); self.stats.compile_wait_ns +|= @intCast(started.untilNow(self.io, .awake).toNanoseconds()); - try self.admit(); - try self.submit(group.specs, group.arena, group.calls, group.progress); + const execution = try self.admit(group.calls); + try self.submit(group.specs, group.arena, group.calls, execution, group.progress); } - /// Retires pending submissions, oldest first, until no program group is - /// pending, so one program group is in flight at a time. A bulk load - /// published before the pending group is retired with it. - fn admit(self: *Loader) !void { + /// Retires pending submissions, oldest first, until the group of + /// `calls` fits the measured room or no program group is pending: a + /// bulk load frees nothing. Once nothing can be retired for it, the + /// room is read again after the freed device memory is given back + /// (`Platform.releaseFreedDeviceMemory`), and the group is admitted + /// whatever its size (`Stats.beyond_room`). Without memory measurement + /// one program group is in flight at a time. Returns the group's output + /// and temporary bytes per device, zero while unmeasured. + fn admit(self: *Loader, calls: []const Call) !DeviceBytes { + var inputs: DeviceBytes = @splat(0); + var execution: DeviceBytes = @splat(0); + var room: ?DeviceBytes = null; + if (self.measured) |direct| { + var temporaries: DeviceBytes = @splat(0); + for (calls) |call| { + const exe = &call.program.exe.?; + for (exe.input_shapes) |shape| addPlacementBytes(&inputs, self.platform, shape); + // The output's memory space is the program's to choose; it is + // charged to the devices like the default space. + addPlacementBytes(&execution, self.platform, exe.output_shapes[0]); + // The group's programs run one after the other, so only the + // largest temporaries are live at once. + const temp = self.compiledTempBytes(exe); + const output_mesh = self.platform.placedShape(exe.output_shapes[0])._sharding.mesh.?; + for (output_mesh.devicesInCanonicalOrder()) |device| temporaries[device.id] = @max(temporaries[device.id], temp); + } + for (&execution, temporaries) |*bytes, temp| bytes.* +|= temp; + room = self.readRoom(direct); + } var retired: usize = 0; - while (self.programGroupPending()) : (retired += 1) try self.retireOldest(true); + while (room == null or !self.fits(&room.?, &inputs, &execution)) : (retired += 1) { + if (!self.programGroupPending()) { + // Nothing is left to retire, but the retired groups' inputs + // may still count as in use: read the room again once the + // devices have them back. The group then goes, beyond the + // room if the room is still short. + if (room != null) { + self.platform.releaseFreedDeviceMemory(); + room = self.readRoom(self.measured.?); + } + break; + } + try self.retireOldest(true); + if (self.measured) |direct| room = self.readRoom(direct); + } + const beyond_room = room != null and !self.fits(&room.?, &inputs, &execution); self.stats.admission_retires += retired; + if (beyond_room) self.stats.beyond_room += 1; + if (retired != 0 or beyond_room) { + // Unmeasured, the group's bytes are not computed: only the retires are logged. + if (self.measured != null) { + const n = self.platform.devices.len; + // A device that stopped reporting its limit shows no room. + const rooms: DeviceBytes = room orelse @splat(0); + const pending = self.pendingExecution(); + // The device the fit is decided on: the short one when the + // group goes beyond the room. + const device = admission.tightest(rooms[0..n], pending[0..n], inputs[0..n], execution[0..n]); + load_log.debug("execute admission: retired={d} before submission {d}: measured=true, beyond_room={}, device {d} room={Bi:.2} pending_execution={Bi:.2} inputs={Bi:.2} execution={Bi:.2}", .{ + retired, + self.stats.submissions, + beyond_room, + device, + rooms[device], + pending[device], + inputs[device], + execution[device], + }); + } else { + load_log.debug("execute admission: retired={d} before submission {d}: measured=false", .{ retired, self.stats.submissions }); + } + } + return execution; + } + + /// Whether a group of `inputs` and `execution` fits beside the pending + /// program groups in `room`, on every device. + fn fits(self: *const Loader, room: *const DeviceBytes, inputs: *const DeviceBytes, execution: *const DeviceBytes) bool { + const n = self.platform.devices.len; + const pending = self.pendingExecution(); + return admission.admits(room[0..n], pending[0..n], inputs[0..n], execution[0..n]); + } + + /// Output and temporary bytes of the pending program groups, per + /// device: what retiring them will take before it frees anything. + fn pendingExecution(self: *const Loader) DeviceBytes { + var sum: DeviceBytes = @splat(0); + var it = self.pending.iterator(); + while (it.nextPtr()) |submission| { + for (&sum, submission.execution) |*bytes, more| bytes.* +|= more; + } + return sum; } /// Whether a program group is pending: only retiring one frees room @@ -458,14 +590,49 @@ pub const Loader = struct { return false; } + /// The room each device reports now; null when one reports no memory + /// limit. + fn readRoom(self: *Loader, direct: *const direct_loader.Loader) ?DeviceBytes { + const n = self.platform.devices.len; + const unallocated = direct.unallocatedBytes(); + var room: DeviceBytes = undefined; + for (room[0..n], self.platform.devices, unallocated[0..n]) |*out, device, device_unallocated| { + const reported = device.memoryStats(); + out.* = admission.room(reported.bytes_limit, reported.bytes_in_use, device_unallocated, admission.reserve_bytes) orelse return null; + self.stats.min_room_seen = @min(self.stats.min_room_seen orelse out.*, out.*); + } + return room; + } + + /// Temporaries the executable takes on each of its devices, from its + /// compiled memory stats (per partition, i.e. per device). Zero when + /// the plugin cannot answer. + fn compiledTempBytes(self: *Loader, exe: *const Exe) u64 { + const api = self.platform.pjrt_api; + const pjrt_executable = exe.exe.executable(api) catch |err| return self.tempUnavailable(err); + defer pjrt_executable.deinit(api); + const stats = pjrt_executable.getCompiledMemoryStats(api) catch |err| return self.tempUnavailable(err); + return stats.temp_size_in_bytes; + } + + fn tempUnavailable(self: *Loader, err: pjrt.ApiError) u64 { + if (!self.stats.temp_unavailable_logged) { + self.stats.temp_unavailable_logged = true; + load_log.debug("executable memory stats unavailable ({s}): temporaries counted as zero", .{@errorName(err)}); + } + return 0; + } + /// One submission over `specs`, which live in `arena`. Takes `arena` and /// `calls` once the submission is published; on failure the caller still - /// owns them. + /// owns them. `execution` is a program group's output and temporary + /// bytes per device (`admit`), zero for a bulk load. fn submit( self: *Loader, specs: []const LoadSpec, arena: std.heap.ArenaAllocator, calls: []Call, + execution: DeviceBytes, progress: std.Progress.Node, ) !void { // One spec per destination, however many parts fill it: its bytes @@ -478,6 +645,7 @@ pub const Loader = struct { .submission = submission, .arena = arena, .calls = calls, + .execution = execution, .logical_bytes = logical_bytes, }); self.stats.submissions += 1; @@ -525,6 +693,15 @@ const Stats = struct { submissions: usize = 0, /// Submissions retired to admit a program group, bulk loads included. admission_retires: usize = 0, + /// The least room a device reported since `init`; null while nothing + /// was measured. + min_room_seen: ?u64 = null, + /// Program groups admitted beyond the measured room since `init`: + /// nothing was left to retire for them. + beyond_room: usize = 0, + /// Whether `tempUnavailable` logged that the plugin gives no compiled + /// memory stats: once per loader. + temp_unavailable_logged: bool = false, /// Times and counts a load from the first `start` since the last /// `wait`; later starts add to it. @@ -558,6 +735,20 @@ const Stats = struct { self.started_at = null; self.compile_wait_ns = 0; } + + /// `deinit`'s debug line. `measured`: whether admission measured the + /// room (`Loader.measured`), printed as `memory_supported`. + fn logAdmission(self: *const Stats, measured: bool) void { + if (self.submissions == 0) return; + load_log.debug("loader admission: submissions={d}, execute_admission_retires={d}, beyond_room={d}, memory_supported={}, min_room_seen={Bi:.2}, reserve={Bi:.2}", .{ + self.submissions, + self.admission_retires, + self.beyond_room, + measured, + self.min_room_seen orelse 0, + admission.reserve_bytes, + }); + } }; /// One published submission: what to wait for, what to run afterwards and @@ -570,6 +761,9 @@ const PendingSubmission = struct { /// Run in order once the reads are done; freed with their inputs when /// the submission is retired. Empty for a bulk load. calls: []Call, + /// Output and temporary bytes per device of a program group, while + /// admission measures (`admit`); zero otherwise and for a bulk load. + execution: DeviceBytes, logical_bytes: usize, }; @@ -681,6 +875,87 @@ const Call = struct { } }; +/// The arithmetic behind program admission: what the devices have left, +/// and whether the next group fits beside the ones still pending. +/// Everything is per device (index = `platform.devices` index) in bytes; +/// nothing here touches a device, so the rules are tested without one. +const admission = struct { + /// Allocator slack per device. BFC rounds every allocation up to 256 bytes; + /// a 64-pack, 16-source, 8-device load is about 1e5 shards, under 32 MiB of + /// rounding, and nothing else runs on the devices during a load. + const reserve_bytes: u64 = 64 << 20; + + /// Bytes the loader may still take on one device: the limit (null when the + /// plugin reports none, as CPU does) minus what is in use, minus its own + /// bytes published but not allocated yet (`unallocated`), minus the + /// reserve. + fn room(limit: ?u64, in_use: u64, unallocated: u64, reserve: u64) ?u64 { + return (limit orelse return null) -| in_use -| unallocated -| reserve; + } + + /// Whether one submission's `inputs` (placements, counted here until they + /// are published) and `execution` (output placement plus the executable's + /// temporaries) fit beside the pending executions on every device. + fn admits(rooms: []const u64, pending: []const u64, inputs: []const u64, execution: []const u64) bool { + for (rooms, pending, inputs, execution) |r, p, i, e| { + if (p +| i +| e > r) return false; + } + return true; + } + + /// The device with the least room left once one submission's `inputs` + /// and `execution` sit beside the pending executions: one that `admits` + /// fails on when the submission does not fit. `rooms` is not empty. + fn tightest(rooms: []const u64, pending: []const u64, inputs: []const u64, execution: []const u64) usize { + var device: usize = 0; + var least: i128 = std.math.maxInt(i128); + for (rooms, pending, inputs, execution, 0..) |r, p, i, e, index| { + const left = @as(i128, r) - (p +| i +| e); + if (left < least) { + least = left; + device = index; + } + } + return device; + } +}; + +test "room subtracts use, unallocated bytes and the reserve, saturating" { + try std.testing.expectEqual(null, admission.room(null, 0, 0, 0)); + try std.testing.expectEqual(55, admission.room(100, 30, 10, 5).?); + // More in use than the limit admits: no room, no underflow. + try std.testing.expectEqual(0, admission.room(100, 130, 0, 0).?); + try std.testing.expectEqual(0, admission.room(100, 90, 0, 20).?); +} + +test "admits needs room on every device" { + try std.testing.expect(admission.admits(&.{ 40, 40 }, &.{ 20, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); + // The second device is one byte short. + try std.testing.expect(!admission.admits(&.{ 40, 34 }, &.{ 20, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); + try std.testing.expect(admission.admits(&.{}, &.{}, &.{}, &.{})); +} + +test "tightest names the device short of room, not the first" { + // Device 0 has less room but holds the group; device 1 is one byte short. + try std.testing.expectEqual(1, admission.tightest(&.{ 30, 34 }, &.{ 0, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); + // Both are short: the one short by more, though it has more room. + try std.testing.expectEqual(1, admission.tightest(&.{ 10, 12 }, &.{ 0, 0 }, &.{ 10, 20 }, &.{ 5, 10 })); + // A need that saturates at maxInt(u64) is short by the most. + try std.testing.expectEqual(1, admission.tightest(&.{ 10, 20 }, &.{ 0, std.math.maxInt(u64) }, &.{ 20, 1 }, &.{ 0, 0 })); + // Equally short: the first. + try std.testing.expectEqual(0, admission.tightest(&.{ 10, 10 }, &.{ 0, 0 }, &.{ 12, 12 }, &.{ 0, 0 })); +} + +/// Adds the bytes `shape` occupies per device: its box on each device it +/// is placed on, as the direct backend's `countSealed` counts them; an +/// empty shape takes none. `shape` places on the platform's devices +/// (`checkPlacement`). +fn addPlacementBytes(out: *DeviceBytes, platform: *const Platform, shape: Shape) void { + if (shape.byteSize() == 0) return; + const grid: ShardGrid = .init(platform, shape); + for (grid.devices) |device| out[device.id] += grid.box_bytes; +} + /// A tensor a program computes, at `path` in the model, the buffer it /// computes into, and its program and inputs. const Computed = struct { @@ -1058,7 +1333,7 @@ const LoaderTestFixture = struct { } }; -test "start publishes the bulk at once and wait runs program groups one at a time" { +test "start publishes the bulk at once and wait runs program groups one at a time without memory measurement" { const allocator = std.testing.allocator; const io = std.testing.io; var fixture: LoaderTestFixture = undefined; @@ -1067,6 +1342,10 @@ test "start publishes the bulk at once and wait runs program groups one at a tim // One program tensor per group. var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); defer loader.deinit(); + // Exercise serial admission regardless of the platform's memory stats: + // decided once the set-up is done. + try loader.awaitSetup(); + loader.measured = null; const Model = struct { value: Tensor, first: Tensor, second: Tensor }; const model: Model = .{ @@ -1086,8 +1365,8 @@ test "start publishes the bulk at once and wait runs program groups one at a tim try std.testing.expectEqual(1, loader.programs.items.len); try loader.wait(); try std.testing.expectEqual(0, loader.pending.len + loader.queued.len + loader.programs.items.len); - // Admission retired the bulk and the first group before it published - // the second. + // Serial admission retired the bulk and the first group before it + // published the second. try std.testing.expectEqual(2, loader.stats.admission_retires); try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); try testing.expectShards(allocator, io, &buffers.first, &LoaderTestFixture.contents, .default); @@ -1163,7 +1442,7 @@ test "tensors of one program and equal args share one compile, and one group run try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); } -test "loader delivers host bindings into pinned host memory" { +test "loader delivers host bindings into pinned host memory without charging the devices" { for ([_]Memory{ .host_pinned, .host_mapped }) |kind| try expectHostDelivery(kind); } @@ -1197,6 +1476,16 @@ fn expectHostDelivery(kind: Memory) !void { // One shard per device, each in that device's pinned host memory space // or, `host_mapped`, a view of pinned memory in its default space. try testing.expectShards(allocator, io, &buffers.host, &LoaderTestFixture.second_contents, kind); + + // Only the tensor in device memory counts against the devices, on both + // sides of the admission ledger. + if (loader.backend.? == .direct) { + const direct = loader.backend.?.direct; + for (direct.sealed_bytes[0..platform.devices.len], direct.allocated_bytes[0..platform.devices.len]) |sealed, *allocated| { + try std.testing.expectEqual(LoaderTestFixture.contents.len, sealed); + try std.testing.expectEqual(sealed, allocated.load(.acquire)); + } + } } test "a program frees its donated and non-donated inputs once" { @@ -1301,6 +1590,8 @@ test "a read failure fails the loader for every later call" { // The broken group first, retired when the next one is admitted. var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); defer loader.deinit(); + try loader.awaitSetup(); + loader.measured = null; const Model = struct { broken: Tensor, value: Tensor }; const model: Model = .{ @@ -1358,6 +1649,38 @@ test "after a failed load every buffer is empty or whole, and deinitBufferized f try std.testing.expectEqual(0, buffers.computed._shards.len); } +test "the direct backend allocates what it was submitted once everything landed" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var placed: DeviceBytes = @splat(0); + // An empty tensor takes no bytes. + addPlacementBytes(&placed, fixture.platform, .init(.{ 0, 4 }, .f32)); + addPlacementBytes(&placed, fixture.platform, fixture.value.shape()); + for (placed[0..fixture.platform.devices.len]) |bytes| try std.testing.expectEqual(LoaderTestFixture.contents.len, bytes); + + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + const Model = struct { value: Tensor, computed: Tensor }; + const model: Model = .{ .value = fixture.value, .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + try loader.wait(); + if (loader.backend.? != .direct) return; + const direct = loader.backend.?.direct; + const n = fixture.platform.devices.len; + const unallocated = direct.unallocatedBytes(); + // Replicated program inputs and bulk outputs occupy every device. + for (direct.sealed_bytes[0..n], direct.allocated_bytes[0..n], unallocated[0..n]) |sealed, *allocated, bytes| { + try std.testing.expectEqual(LoaderTestFixture.contents.len * 2, sealed); + try std.testing.expectEqual(sealed, allocated.load(.acquire)); + try std.testing.expectEqual(0, bytes); + } +} + test "init checks every option and returns InvalidOptions" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -1685,6 +2008,7 @@ test "loader assembles concatenated and padded sources on every placement and me // Rows over the devices (b and c straddle shards on four), every // row split over them, and every byte on each of them. inline for (.{ .{ .row = .model }, .{ .col = .model }, .{} }) |partitioning| { + const shards: u64 = if (@typeInfo(@TypeOf(partitioning)).@"struct".field_names.len == 0) 1 else device_count; for ([_]Memory{ .default, .host_pinned, .host_mapped }) |memory| { if (!backend.supportsMemory(platform, memory)) continue; const view = store.view(); @@ -1715,6 +2039,17 @@ test "loader assembles concatenated and padded sources on every placement and me try testing.expectShards(allocator, io, &buffers.plain, &fixture.plain, memory); // Each destination once, padding included. try std.testing.expectEqual(@as(usize, 256 + 128 + 96 + 64), loader.stats.bytes_loaded); + // A destination is allocated once per device, whatever its + // parts; host memory counts on neither side. + if (loader.backend.? == .direct) { + const direct = loader.backend.?.direct; + const n = platform.devices.len; + for (direct.sealed_bytes[0..n], direct.allocated_bytes[0..n]) |sealed, *allocated| try std.testing.expectEqual(sealed, allocated.load(.acquire)); + for (devices.physical.devices_in_canonical_order) |device| { + const expected: u64 = if (memory.isHost()) 0 else (256 + 128 + 64) / shards + 96; + try std.testing.expectEqual(expected, direct.sealed_bytes[device.id]); + } + } } } } From 1aa2d78039fb0596ec95205a940d33c6cd90d571 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 5 Nov 2026 09:35:28 +0100 Subject: [PATCH 089/103] zml/io: interleave host arenas over NUMA nodes The page-backed DMA arenas (CUDA, oneAPI and CPU) are now interleaved with mbind over the host's memory-bearing NUMA nodes, the ones /sys/devices/system/node/has_memory lists, which leaves out device-coherent HBM nodes, narrowed to the nodes of the platform's devices. The set is masked by the task's cpuset and memory policy: arenas stay within the nodes both allow; among several, they go to the devices' nodes. When a device reports no node (the CPU plugin reports none), or none of theirs is allowed, they interleave over every allowed node. With a single allowed memory-bearing node there is nothing to choose and the kernel places them. If the kernel refuses the policy, it is logged once at warn and every later arena stays unplaced. ROCm and Furiosa's pjrt_host arenas keep their per-node allocation paths. On four GB300 over two nodes, warm replicated DeepSeek-V4-Flash loaded in 4.35 s from one pool interleaved over both against 4.92 s from node-local pools. On two GB300 of one Grace node, Qwen3.8-27B-FP8 loaded in 1.265 s with a task policy holding the arenas to that node against 1.64 s interleaved over both nodes. The arena debug line now reports the placement and its node mask. Co-Authored-By: Claude Opus 5.5 --- zml/io/host_memory.zig | 189 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 179 insertions(+), 10 deletions(-) diff --git a/zml/io/host_memory.zig b/zml/io/host_memory.zig index 1f684658..a8943c36 100644 --- a/zml/io/host_memory.zig +++ b/zml/io/host_memory.zig @@ -2,13 +2,17 @@ //! its workspace for its whole life: calibration maps its ring there, the //! backend grows the set to what its load holds and carves it, and its //! `destroy` releases every arena. Callers never supply a workspace, a -//! budget or a calibration result. +//! budget, a NUMA policy or a calibration result. //! //! The memory depends on the target: CUDA and oneAPI take page-backed arenas //! registered with PJRT (DmaMap), ROCm and Furiosa PJRT `pinned_host` //! buffers, and CPU plain pages, unregistered. The arenas' kind and the //! transfer path are separate decisions: CPU loads through the direct -//! backend too. ROCm balances whole arenas across its devices' host nodes. +//! backend too. When the task's cpuset and memory policy allow several +//! memory-bearing NUMA nodes, page-backed arenas interleave over those of the +//! platform's devices (all of them when a device's node is unknown or none of +//! theirs is allowed); ROCm balances whole arenas across its devices' host +//! nodes. //! `Workspace.mapped_bytes_ceiling` (16 GiB) is a safety guard, not a //! target. A failed allocation or registration, or a growth past the //! ceiling, is logged with its cause and returns `OutOfMemory`. @@ -61,21 +65,29 @@ const ArenaKind = union(enum) { /// transfers at 11.01 GiB/s instead of 26.90 (eight DMA caps). pages: struct { platform: ?*const Platform, + /// Zero leaves placement to the kernel. Automatic placement can + /// fall back to zero if the kernel refuses it. + numa_mask: u64, }, const transparent_huge_page_size = 2 * 1024 * 1024; + const mpol_interleave = 3; - fn init(allocator: Allocator, platform: *const Platform) Allocator.Error!ArenaKind { + fn init( + allocator: Allocator, + io: std.Io, + platform: *const Platform, + ) Allocator.Error!ArenaKind { return switch (platform.target) { - .cuda, .oneapi => .{ .pages = .{ .platform = platform } }, - .cpu => .{ .pages = .{ .platform = null } }, + .cuda, .oneapi => .{ .pages = .{ .platform = platform, .numa_mask = pagesNumaMask(allocator, io, platform) } }, + .cpu => .{ .pages = .{ .platform = null, .numa_mask = pagesNumaMask(allocator, io, platform) } }, // Furiosa backs `.host_pinned` with SDK pinned (dma-buf) memory // from `/dev/dma_heap/system`, which the SDK recognizes by // address and transfers without pinning pages on every request. .rocm, .furiosa => { // PJRT-pinned allocations choose placement through the device's - // host memory space. Retain one allocation path per reported - // node; the blocks stay unpartitioned. + // host memory space, not our mbind policy. Retain one allocation + // path per reported node; the blocks stay unpartitioned. var host_nodes: std.ArrayList(HostNode) = try .initCapacity(allocator, @max(platform.devices.len, 1)); devices: for (platform.devices, 0..) |device, device_index| { std.debug.assert(device.memory(.host_pinned) != null); @@ -169,6 +181,7 @@ const ArenaKind = union(enum) { return error.OutOfMemory; }; const data = ptr[0..bytes]; + place(&pages.numa_mask, data); adviseHugePages(data); if (pages.platform) |p| p.pjrt_client.dmaMap(p.pjrt_api, data) catch |err| { log.err("DMA registration failed for {Bi:.2} of host memory: {s}", .{ bytes, @errorName(err) }); @@ -176,8 +189,10 @@ const ArenaKind = union(enum) { return error.OutOfMemory; }; const finished: std.Io.Timestamp = .now(io, .awake); - log.debug("DMA arena kind={s} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ + log.debug("DMA arena kind={s} placement={s} nodes=0x{x} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ if (pages.platform != null) "dma_map" else "pageable", + if (pages.numa_mask == 0) "unplaced" else "interleave", + pages.numa_mask, @intFromPtr(data.ptr), bytes, @as(f64, @floatFromInt(elapsedNanoseconds(started, finished))) / std.time.ns_per_ms, @@ -217,6 +232,35 @@ const ArenaKind = union(enum) { return alignment.max(.fromByteUnits(transparent_huge_page_size)); } + /// Interleaves `data` over `numa_mask`'s nodes; on refusal, warns and + /// zeroes the mask, so later arenas stay unplaced too. + fn place(numa_mask: *u64, data: []u8) void { + const mask = numa_mask.*; + if (mask == 0) return; + if (comptime builtin.os.tag != .linux) return; + + const node_mask: [1]u64 = .{mask}; + const highest_node: usize = 63 - @clz(mask); + const rc = std.os.linux.syscall6( + .mbind, + @intFromPtr(data.ptr), + data.len, + mpol_interleave, + @intFromPtr(&node_mask), + // Linux get_nodes() decrements maxnode before copying the mask; + // raw callers include the same extra sentinel bit as libnuma. + highest_node + 2, + 0, + ); + if (std.os.linux.errno(rc) == .SUCCESS) return; + log.warn("NUMA placement of DMA arenas over nodes 0x{x} refused ({s}); leaving them unplaced", .{ + mask, + @tagName(std.os.linux.errno(rc)), + }); + // Automatic placement is not retried: the arenas stay unplaced. + numa_mask.* = 0; + } + fn adviseHugePages(data: []u8) void { if (comptime builtin.os.tag != .linux) return; if (data.len < transparent_huge_page_size) return; @@ -259,7 +303,7 @@ pub const Workspace = struct { return .{ .allocator = allocator, .io = io, - .kind = try .init(allocator, platform), + .kind = try .init(allocator, io, platform), }; } @@ -348,17 +392,142 @@ pub const Workspace = struct { return .{ .allocator = allocator, .io = io, - .kind = .{ .pages = .{ .platform = null } }, + .kind = .{ .pages = .{ .platform = null, .numa_mask = 0 } }, }; } }; pub const Block = []u8; +/// Bits of `/sys/devices/system/node/has_memory`, or zero when unreadable. +/// Nodes 64 and above cannot be represented and are dropped. +/// Device-coherent HBM nodes are not candidates for host pages: the measured +/// four-GB300 topology had 34 NUMA nodes but only two with host memory. +fn memoryNodeMask(allocator: Allocator, io: std.Io) u64 { + if (comptime builtin.os.tag != .linux) return 0; + const contents = std.Io.Dir.cwd().readFileAlloc( + io, + "/sys/devices/system/node/has_memory", + allocator, + .limited(4096), + ) catch return 0; + defer allocator.free(contents); + return parseNodeList(contents); +} + +/// The nodes the task's memory policy allows (`numactl --membind`, +/// `--interleave`, `--preferred`); every node under the default policy. +/// An arena placed outside them would defeat the policy: `mbind` on a +/// range overrides `set_mempolicy`. +fn taskMemoryPolicyMask() u64 { + if (comptime builtin.os.tag != .linux) return std.math.maxInt(u64); + var mode: c_int = 0; + var nodes: [1]u64 = .{0}; + const rc = std.os.linux.syscall5(.get_mempolicy, @intFromPtr(&mode), @intFromPtr(&nodes), 64, 0, 0); + if (std.os.linux.errno(rc) != .SUCCESS or mode == 0 or nodes[0] == 0) return std.math.maxInt(u64); + return nodes[0]; +} + +/// The nodes the task's cpuset lets it allocate on (`Mems_allowed`), or +/// every node when unknown. `mbind` refuses a mask with none of them. +fn cpusetMemsMask() u64 { + if (comptime builtin.os.tag != .linux) return std.math.maxInt(u64); + const mpol_f_mems_allowed = 4; + var nodes: [1]u64 = .{0}; + const rc = std.os.linux.syscall5(.get_mempolicy, 0, @intFromPtr(&nodes), 64, 0, mpol_f_mems_allowed); + if (std.os.linux.errno(rc) != .SUCCESS or nodes[0] == 0) return std.math.maxInt(u64); + return nodes[0]; +} + +/// The nodes page-backed arenas interleave over (`placementMask`): the +/// memory-bearing nodes the task's cpuset and memory policy allow, narrowed +/// to the nodes of the platform's devices. +fn pagesNumaMask(allocator: Allocator, io: std.Io, platform: *const Platform) u64 { + const candidates = memoryNodeMask(allocator, io) & cpusetMemsMask() & taskMemoryPolicyMask(); + return placementMask(candidates, deviceNodeMask(platform)); +} + +/// The nodes of the platform's devices (`Device.numaNode`), or null when a +/// device reports none (the CPU plugin never does) or a node past 63. +fn deviceNodeMask(platform: *const Platform) ?u64 { + var mask: u64 = 0; + for (platform.devices) |device| { + const node = device.numaNode() orelse return null; + if (node >= 64) return null; + mask |= @as(u64, 1) << @intCast(node); + } + return if (mask == 0) null else mask; +} + +/// Zero leaves placement to the kernel: with one candidate node or none +/// there is nothing to choose, and the task's policy or the only node decides. +/// Among several candidates, the arenas go to the devices' nodes: an arena on +/// another node crosses the inter-socket link on every H2D copy. Two GB300 +/// on one Grace node loaded Qwen3.8-27B-FP8 in 1.265 s with a task policy +/// holding the arenas to that node, against 1.64 s interleaved over both +/// nodes. Devices on several nodes share one pool interleaved over theirs: +/// four GB300 on two nodes loaded replicated DeepSeek-V4-Flash in 4.35 s +/// against 4.92 s with per-node pools. When a device's node is unknown, or +/// none is a candidate, the arenas interleave over every candidate, which no +/// single node beat everywhere. A single node is placed too, since unplaced +/// CUDA arenas follow the registering thread's node; the kernel still falls +/// back when it is full. +fn placementMask(candidates: u64, device_nodes: ?u64) u64 { + if (@popCount(candidates) < 2) return 0; + const near = candidates & (device_nodes orelse candidates); + return if (near != 0) near else candidates; +} + +/// Parses a kernel node list ("0-1", "0,2-3") into a mask; zero on any error. +fn parseNodeList(text: []const u8) u64 { + var mask: u64 = 0; + var ranges = std.mem.tokenizeAny(u8, text, ", \n"); + while (ranges.next()) |range| { + var ends = std.mem.splitScalar(u8, range, '-'); + const first = std.fmt.parseInt(usize, ends.first(), 10) catch return 0; + const last = if (ends.next()) |end| + std.fmt.parseInt(usize, end, 10) catch return 0 + else + first; + if (last < first or ends.next() != null) return 0; + var node = first; + while (node <= last) : (node += 1) { + if (node < 64) mask |= @as(u64, 1) << @intCast(node); + } + } + return mask; +} + fn elapsedNanoseconds(started: std.Io.Timestamp, finished: std.Io.Timestamp) u64 { return @intCast(@max(started.durationTo(finished).nanoseconds, 0)); } +test "automatic NUMA placement keeps the devices' nodes among several candidates" { + // One candidate or none: the kernel places. + try std.testing.expectEqual(@as(u64, 0), placementMask(0, 0b10)); + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1)); + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1000, null)); + // Devices on one of two nodes: that node; on both: both. + try std.testing.expectEqual(@as(u64, 0b10), placementMask(0b11, 0b10)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11)); + // Unknown device nodes, or none among the candidates: every candidate. + try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b100)); + // A cpuset of nodes 0 and 2 of four, devices on node 1: the cpuset's + // nodes, as `pagesNumaMask` narrows the candidates to them first. + try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10)); +} + +test "parseNodeList accepts kernel node lists" { + try std.testing.expectEqual(@as(u64, 0b11), parseNodeList("0-1\n")); + try std.testing.expectEqual(@as(u64, 0b1101), parseNodeList("0,2-3")); + try std.testing.expectEqual(@as(u64, 0b1), parseNodeList("0")); + try std.testing.expectEqual(@as(u64, 0), parseNodeList("")); + try std.testing.expectEqual(@as(u64, 0), parseNodeList("1-0")); + try std.testing.expectEqual(@as(u64, 0), parseNodeList("x")); + try std.testing.expectEqual(@as(u64, 0b1), parseNodeList("0,64")); +} + test "Workspace capacity counts whole blocks under the mapped ceiling, and carve refuses one more" { const allocator = std.testing.allocator; var workspace: Workspace = .initForTesting(allocator, std.testing.io); From 212896b88a8b006278efa42a1e38c0155cbf01e2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 14:46:16 +0100 Subject: [PATCH 090/103] zml/io: place the host arenas for direct reads from a set-up prediction A load that reads at least four fifths of its checkpoint's bytes past the page cache (ZML_DIRECT_IO=on, or a cold checkpoint under auto) interleaves its page-backed arenas over every allowed node instead of the devices' nodes: the drives write into them, and interleaving splits that traffic, and the copies out of them, between the nodes' memory. Before calibration maps the first arena, the direct backend predicts how the files the registry reads for the checkpoint (safetensors.checkpointFiles) will read, each with the VFS's own rule (VFS.predictDirectIo), and a debug line reports the prediction. Workspace.init takes the prediction as direct_reads. Each of those files then reads as predicted: the planner keeps the answers by real path, as the registry names the files, and under auto decides a file predicted direct under on and one predicted buffered under off, rather than sampling its residency again; under on, which samples nothing, every file keeps on. The prediction's sample queues the read of each page it misses and of the kernel's readahead, and a second sample of those pages once the reads have landed finds the file cached: on the GB300 host below (64 KiB pages), sampling again when planning read 3 of the 330 files of five cold loads buffered, at 75 to 97% cached. A file whose residency changes before it is planned costs speed, never bytes, and under on a filesystem that refuses the flag still reads it buffered and logs why. Loader.init copies the checkpoint's path for that set-up when it is local and the VFS may read it direct, so it may also return OutOfMemory; the set-up task frees the copy, canceled or not. Under zio the prediction takes a pool worker of its own (zio.blockInPlaceReserved), as Compiler.compile does. zio hands the file operations its event loop does not run (a real path under io_uring, every one under epoll) to its blocking pool, which adds a worker for such a job only when none is idle and its queue holds twice as many jobs as it has workers, and the compiles hold theirs for their whole run. Queued there, llmd's set-up on the GB300 host below waited for the first compile to end: it placed its arenas 49.8 and 51.9 s after llmd started, against 1.6 and 1.9 s on a worker of its own, and with two compiles of 28 s the load of one process in two waited about 0.3 s for it. On two GB300 of one Grace node, over an md RAID0 of four NVMe drives two on each node, Qwen3.8-27B-FP8 forced direct loaded in 1.752 s with the arenas interleaved over both nodes, against 1.873 s with them on the devices' node, in the same session: the drives read as fast either way (1.90 against 1.87 ms per 16 MiB read), and staging a read from the arenas took 0.52 against 0.64 ms. Cold under auto, from a private copy on the same RAID evicted before each load, it loaded in 1.658 against 1.772 s, and in 1.690 s on the original branch, in another session: a 16 MiB read took 1.87 ms either way, and staging it 0.49 against 0.69 ms. Each time is the median of five loads. That is about 4.2 and 4.0 ms per GiB saved on direct reads, against 10 to 16 ms per GiB that spreading costs reads through the page cache, so spreading pays from about 0.71 to 0.80 of the bytes read direct, and four fifths, at the top of that range, leans to the devices' nodes. Page-cached, the checkpoint keeps loading from the devices' node. Co-Authored-By: Claude Opus 5.5 --- zml/io/backend.zig | 7 + zml/io/direct_loader.zig | 309 +++++++++++++++++++++++++++++++++++++-- zml/io/host_memory.zig | 86 +++++++---- zml/io/loader.zig | 73 +++++++-- 4 files changed, 425 insertions(+), 50 deletions(-) diff --git a/zml/io/backend.zig b/zml/io/backend.zig index dceaeab5..722c4e0a 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -55,6 +55,13 @@ pub const Options = struct { read_hints: VFS.ReadHints = .local, dma_block_size: ?usize = null, direct_io: VFS.DirectIo = .auto, + /// The loader's checkpoint (loader.zig, `Loader.Options.checkpoint`) + /// when it is local and the VFS may read it direct (`readAlignment` is + /// not 0), else null; owned by the loader's set-up task. The direct + /// backend predicts from it how the checkpoint's files read, places its + /// arenas for that, and reads each of those files as predicted + /// (`direct_loader.Loader.create`); it keeps nothing of the path. + checkpoint: ?[]const u8 = null, /// The blocks calibration may select: `dma_block_size`, else /// `dma_calibration.default_block_sizes`. diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 5938fa86..28a69375 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -39,7 +39,11 @@ //! its range, into blocks that are whole units, and its transfers address //! the widened read. A job's range is at most a request less two units, so //! its widened read still fits its credit's slot. Every other file is read -//! exact. +//! exact. The arenas are placed for how the checkpoint reads, predicted +//! before calibration maps the first (`Planner.Reads.predict`): over every +//! allowed node when at least four fifths of its bytes will read direct +//! (host_memory.zig). The files predicted then read as predicted, and every +//! other file as its own decision says. //! //! Destinations. A part's source bytes map to its destination's logical //! offsets, which the destination's `ShardGrid` places on the devices by @@ -85,6 +89,7 @@ const builtin = @import("builtin"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); const VFS = @import("vfs"); +const zio = @import("zio"); const Buffer = @import("../buffer.zig").Buffer; const pjrtx = @import("../pjrtx.zig"); @@ -184,7 +189,31 @@ pub const Loader = struct { const self = try allocator.create(Loader); errdefer allocator.destroy(self); - var workspace: host_memory.Workspace = try .init(allocator, io, platform); + var reads: Planner.Reads = .init(io, opts.direct_io); + errdefer reads.deinit(allocator); + // The arenas are placed for how the checkpoint reads before + // calibration maps the first: registered pages never move. + const probe_started: std.Io.Timestamp = .now(io, .awake); + const predicted = switch (platform.io_impl) { + .threaded => try reads.predict(allocator, opts.checkpoint), + // zio hands a file operation its event loop does not run (a + // real path; any, where it falls back to epoll) to a pool + // worker, and the compiles hold theirs for their whole run + // (`Compiler.compile`): the prediction takes a worker of its own + // rather than queue behind them, so the set-up overlaps them. + .zio => try zio.blockInPlaceReserved(Planner.Reads.predict, .{ &reads, allocator, opts.checkpoint }), + }; + if (opts.checkpoint != null) { + const probe_ns: u64 = @intCast(@max(probe_started.untilNow(io, .awake).nanoseconds, 0)); + load_log.debug("checkpoint reads predicted: files={d}, direct={Bi:.2} of {Bi:.2}, direct_reads={}, probe_ms={d:.3}", .{ + predicted.files, + predicted.direct_bytes, + predicted.bytes, + predicted.direct(), + @as(f64, @floatFromInt(probe_ns)) / std.time.ns_per_ms, + }); + } + var workspace: host_memory.Workspace = try .init(allocator, io, platform, predicted.direct()); errdefer workspace.deinit(); // At debug when canceled (`zml.io.next.Loader.deinit` cancels a // set-up no `start` waited for) or out of memory (the workspace logs @@ -193,7 +222,6 @@ pub const Loader = struct { load_log.failure(err, "calibrate DMA: target={t}", .{platform.target}); return err; }; - const reads: Planner.Reads = .init(io, opts.direct_io); const request_size = backend.requestSize(opts.read_hints.read_chunk_size, block_size, reads.alignment).?; const blocks_per_request = request_size / block_size; // The DMA stage of every device, kept mapped beside the source @@ -439,6 +467,7 @@ pub const Loader = struct { self.allocator.destroy(slot.*); } self.source_slots.deinit(self.allocator); + self.reads.deinit(self.allocator); std.debug.assert(self.credits.inUse(self.io) == 0); for (self.pumps) |*device_pump| { std.debug.assert(device_pump.active_pieces == 0); @@ -1061,11 +1090,12 @@ const Planner = struct { }; /// How each file is read: aligned when it is local and the VFS that - /// opens it reads it directly under `policy`, decided once per file and - /// kept in its slot. A remote file is read exact without being opened - /// here. Without a VFS (`io` is none, or no alignment) every file takes - /// `alignment`. A gather job reads files other than its plan's, so it - /// asks here for each. + /// opens it reads it directly under `policy`, or as `predict` answered + /// for it, decided once per file and kept in its slot. A remote file is + /// read exact without being opened here. Without a VFS (`io` is none, or + /// no alignment) every file takes `alignment`. A gather job reads files + /// other than its plan's, so it asks here for each. `predict` tells, + /// before any file is planned, how the checkpoint's files will read. /// Widening buffered reads is not free: a widened read served from the /// page cache costs the DMA sources their block alignment for nothing, /// and warm replicated Llama on four GB300 lost 3-6% when its first @@ -1080,23 +1110,45 @@ const Planner = struct { /// Every read of a file read directly starts and ends at a multiple /// of this; 0 reads every file exact. alignment: usize, + /// The files `predict` answered for, by real path as the registry + /// names them (`safetensors.Tensor.file_uri`), each with the policy + /// that decides it as answered: under `auto`, `on` for a file that + /// reads direct and `off` for one that reads buffered. Under `on`, + /// which samples nothing, every file keeps `on`, and a filesystem + /// refusing the flag is logged when the file is planned. Owned until + /// `deinit`. + predicted: std.StringHashMapUnmanaged(VFS.DirectIo) = .empty, fn init(io: std.Io, policy: VFS.DirectIo) Reads { const alignment = backend.readAlignment(io, policy); return .{ .io = io, .vfs = if (alignment != 0) VFS.fromIo(io) else null, .policy = policy, .alignment = alignment }; } + fn deinit(self: *Reads, allocator: std.mem.Allocator) void { + var paths = self.predicted.keyIterator(); + while (paths.next()) |path| allocator.free(path.*); + self.predicted.deinit(allocator); + } + /// The file's alignment, decided the first time it is asked: a - /// local file is opened here. One that fails to open is read exact: + /// local file is opened here, and decided as `predict` answered for + /// it, else under `policy`. One that fails to open is read exact: /// its reads fail with the same error (`LazyOnce`), which `readSlot` /// logs with the path. fn alignmentOf(self: Reads, slot: *SourceSlot) error{Canceled}!usize { const vfs = self.vfs orelse return self.alignment; if (slot.alignment) |known| return known; + // A predicted file is decided as predicted, not sampled again: + // the prediction's sample queued the read of each page it missed + // and of the kernel's readahead, and a second sample of those + // pages once the reads have landed finds the file cached + // (`VFS.predictDirectIo`). Under `on`, a filesystem refusing the + // flag still reads it buffered. + const policy = self.predicted.get(slot.uri) orelse self.policy; const direct = vfs.isLocal(slot.uri) and vfs.useDirectIo(slot.ensure(self.io) catch |err| switch (err) { error.Canceled => return error.Canceled, else => return 0, - }, self.policy); + }, policy); const alignment = if (direct) self.alignment else 0; slot.alignment = alignment; return alignment; @@ -1108,6 +1160,76 @@ const Planner = struct { const vfs = self.vfs orelse return self.alignment; return if (vfs.isLocal(uri)) self.alignment else 0; } + + /// How the checkpoint's local safetensors files will read, asked + /// once, before any is planned, to place the arenas: each as + /// `VFS.predictDirectIo` answers under `policy`, an answer kept in + /// `predicted` for `alignmentOf` to decide the file by. The files + /// are those `TensorRegistry.fromPath` reads for the checkpoint + /// (`safetensors.checkpointFiles`, which reads its index). Nothing + /// without a VFS, for a checkpoint that is not local, or for files + /// that do not resolve or open: the registry reports those. Keeps + /// no file open, and logs nothing. A wrong prediction (residency + /// changing before the file is planned) costs speed, never bytes. + fn predict(self: *Reads, allocator: std.mem.Allocator, checkpoint: ?[]const u8) error{ Canceled, OutOfMemory }!Prediction { + std.debug.assert(self.predicted.count() == 0); + const vfs = self.vfs orelse return .{}; + const path = checkpoint orelse return .{}; + if (!vfs.isLocal(path)) return .{}; + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const dir, const names = safetensors.checkpointFiles(arena.allocator(), self.io, path) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.OutOfMemory => return error.OutOfMemory, + else => return .{}, + }; + defer dir.close(self.io); + var prediction: Prediction = .{}; + var real_path_buffer: [std.fs.max_path_bytes]u8 = undefined; + for (names) |name| { + const real_path_len = dir.realPathFile(self.io, name, &real_path_buffer) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => continue, + }; + const real_path = real_path_buffer[0..real_path_len]; + // A symlink and the file it leads to are sampled once. + if (self.predicted.contains(real_path)) continue; + const file = try vfs.predictDirectIo(dir, name, self.policy) orelse continue; + const owned = try allocator.dupe(u8, real_path); + errdefer allocator.free(owned); + try self.predicted.putNoClobber(allocator, owned, if (file.direct or self.policy == .on) .on else .off); + prediction.add(file); + } + return prediction; + } + + /// The bytes of a checkpoint's local safetensors files, and of those + /// that will read direct (`predict`). + const Prediction = struct { + files: usize = 0, + bytes: u64 = 0, + direct_bytes: u64 = 0, + + fn add(self: *Prediction, file: VFS.DirectIoPrediction) void { + self.files += 1; + self.bytes += file.size; + if (file.direct) self.direct_bytes += file.size; + } + + /// Whether the arenas are placed for direct reads + /// (`host_memory.Workspace.init`): files holding at least four + /// fifths of the bytes read direct. In measurements on two GB300 + /// of one Grace node loading Qwen3.8-27B-FP8 (28.75 GiB), + /// spreading the arenas over both nodes saved direct reads about + /// 4.2 ms per GiB forced direct and 4.0 cold under auto, and cost + /// reads through the page cache 10 to 16 ms per GiB: it pays from + /// about 0.71 to 0.80 of the bytes read direct, and four fifths, + /// at the top of that range, leans to the devices' nodes, whose + /// mistake costs less per GiB. + fn direct(self: Prediction) bool { + return self.bytes != 0 and self.direct_bytes * 5 >= self.bytes * 4; + } + }; }; /// Plans `parts` one file at a time and publishes each plan as soon as @@ -3283,6 +3405,13 @@ fn fixedReads(alignment: usize) Planner.Reads { return .{ .io = undefined, .vfs = null, .policy = .off, .alignment = alignment }; } +/// What a loader's `Planner.Reads` predicts for `checkpoint` under `policy`. +fn testPrediction(io: std.Io, policy: VFS.DirectIo, checkpoint: ?[]const u8) !Planner.Reads.Prediction { + var reads: Planner.Reads = .init(io, policy); + defer reads.deinit(std.testing.allocator); + return reads.predict(std.testing.allocator, checkpoint); +} + /// `Planner.preparePlan` with every file read at `alignment`, kept in /// `arena`; the testing allocator checks the cut for leaks. fn prepareTestPlan(arena: std.mem.Allocator, parts: []Part, order: []const usize, config: Planner.Config, alignment: usize) !*Plan { @@ -3362,6 +3491,166 @@ test "the planner opens only local files to ask about direct I/O" { try std.testing.expectEqual(@as(usize, if (vfs.useDirectIo(file, .on)) 4096 else 0), alignment); } +test "the checkpoint's reads are predicted as the VFS will decide the files its registry reads" { + if (VFS.direct_io_alignment == 0) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const long: [2 * VFS.direct_io_alignment + 100]u8 = @splat(7); + try tmp.dir.writeFile(io, .{ .sub_path = "a.safetensors", .data = &long }); + try tmp.dir.writeFile(io, .{ .sub_path = "b.safetensors", .data = &long }); + try tmp.dir.writeFile(io, .{ .sub_path = "c.safetensors", .data = long[0..100] }); + // The index lists a, b, c and a symlink to a; the registry does not + // read the consolidated file beside them (Mistral's format). The files + // hold no header: none is read. + try tmp.dir.symLink(io, "a.safetensors", "alias.safetensors", .{}); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"x\":\"a.safetensors\",\"y\":\"b.safetensors\",\"z\":\"c.safetensors\",\"w\":\"alias.safetensors\"}}" }); + try tmp.dir.writeFile(io, .{ .sub_path = "consolidated.safetensors", .data = &long }); + // A snapshot whose file links into blobs, as a Hugging Face cache's do. + const blob: [8 * 1024]u8 = @splat(9); + try tmp.dir.createDir(io, "blobs", .default_dir); + try tmp.dir.writeFile(io, .{ .sub_path = "blobs/x", .data = &blob }); + try tmp.dir.createDir(io, "snapshot", .default_dir); + try tmp.dir.symLink(io, "../blobs/x", "snapshot/model.safetensors", .{}); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; + var path_buffer: [std.fs.max_path_bytes + 64]u8 = undefined; + + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + // A remote scheme: nothing is predicted for a checkpoint there. + try vfs.registerBackend("remote", .{ .io = io }); + const vfs_io = vfs.io(); + + // The listed files as fresh handles decide them under `on`: c, shorter + // than a unit, never direct. + var expected: Planner.Reads.Prediction = .{}; + const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); + defer dir.close(vfs_io); + for ([_][]const u8{ "a.safetensors", "b.safetensors", "c.safetensors" }) |name| { + const file = try dir.openFile(vfs_io, name, .{}); + defer file.close(vfs_io); + const size = (try file.stat(vfs_io)).size; + expected.files += 1; + expected.bytes += size; + if (vfs.useDirectIo(file, .on)) expected.direct_bytes += size; + } + try std.testing.expectEqual(@as(u64, 2 * long.len + 100), expected.bytes); + + var on_reads: Planner.Reads = .init(vfs_io, .on); + defer on_reads.deinit(allocator); + // The symlink is predicted once, with a. + try std.testing.expectEqual(expected, try on_reads.predict(allocator, root)); + // Each answer is kept by the file's real path, none for the + // consolidated file. Under `on` each keeps `on`, c too: planning + // decides it, and it reads buffered. + try std.testing.expectEqual(@as(usize, 3), on_reads.predicted.count()); + try std.testing.expectEqual(VFS.DirectIo.on, on_reads.predicted.get(try std.fmt.bufPrint(&path_buffer, "{s}/c.safetensors", .{root})).?); + try std.testing.expectEqual(expected, try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/model.safetensors.index.json", .{root}))); + const one = try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/a.safetensors", .{root})); + try std.testing.expectEqual(@as(usize, 1), one.files); + try std.testing.expectEqual(@as(u64, long.len), one.bytes); + const snapshot = try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/snapshot", .{root})); + try std.testing.expectEqual(@as(usize, 1), snapshot.files); + try std.testing.expectEqual(@as(u64, blob.len), snapshot.bytes); + + // Just written, so cached: none reads direct under `auto`, and each is + // kept as buffered. + var auto_reads: Planner.Reads = .init(vfs_io, .auto); + defer auto_reads.deinit(allocator); + const cached = try auto_reads.predict(allocator, root); + try std.testing.expectEqual(@as(usize, 3), cached.files); + try std.testing.expectEqual(@as(u64, 0), cached.direct_bytes); + var answers = auto_reads.predicted.valueIterator(); + while (answers.next()) |answer| try std.testing.expectEqual(VFS.DirectIo.off, answer.*); + + // Nothing to predict: no direct I/O, no VFS, a remote or no checkpoint, + // or none there. + const nothing = [_]Planner.Reads.Prediction{ + try testPrediction(vfs_io, .off, root), + try testPrediction(io, .on, root), + try testPrediction(vfs_io, .on, "remote://bucket/model"), + try testPrediction(vfs_io, .on, null), + try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/missing", .{root})), + try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/blobs", .{root})), + }; + for (nothing) |prediction| try std.testing.expectEqual(@as(usize, 0), prediction.files); +} + +test "a file predicted to read direct is planned direct, though the prediction's sample cached it" { + if (comptime VFS.direct_io_alignment == 0) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // A snapshot whose file links into blobs, as a Hugging Face cache's + // does. 64 KiB: every page of it is sampled, on 4 KiB pages too. + const data: [64 * 1024]u8 = @splat(7); + const blobs = try tmp.dir.createDirPathOpen(io, "blobs", .{}); + defer blobs.close(io); + try testing.writeSafetensorsFile(allocator, io, blobs, "x", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{data.len}, .bytes = &data }, + }); + try tmp.dir.createDir(io, "snapshot", .default_dir); + try tmp.dir.symLink(io, "../blobs/x", "snapshot/model.safetensors", .{}); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; + var path_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + const checkpoint = try std.fmt.bufPrint(&path_buffer, "{s}/snapshot", .{root}); + + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + const vfs_io = vfs.io(); + // The planner's slot takes the registry's name for the file: the blob. + var registry: safetensors.TensorRegistry = try .fromPath(allocator, vfs_io, checkpoint); + defer registry.deinit(); + const uri = registry.tensors.get("w").?.file_uri; + { + const file = try blobs.openFile(io, "x", .{}); + defer file.close(io); + try file.sync(io); + _ = std.os.linux.fadvise(file.handle, 0, 0, std.os.linux.POSIX_FADV.DONTNEED); + } + + var reads: Planner.Reads = .init(vfs_io, .auto); + defer reads.deinit(allocator); + const predicted = try reads.predict(allocator, checkpoint); + // A filesystem that kept the file cached (tmpfs) or refuses direct + // reads says nothing here. + if (predicted.direct_bytes == 0) return error.SkipZigTest; + // Once the sample's reads landed, a probe of its own finds the file + // cached (a kernel that queues no read for a missed sample shows + // nothing). + try io.sleep(.fromMilliseconds(250), .awake); + { + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, uri, .{ .mode = .read_only }); + defer file.close(vfs_io); + if (vfs.useDirectIo(file, .auto)) return error.SkipZigTest; + } + + // Planning reads it as predicted: direct, the handle decided so. + var slot: SourceSlot = .{ .uri = uri }; + defer slot.deinit(vfs_io); + try std.testing.expectEqual(reads.alignment, try reads.alignmentOf(&slot)); + try std.testing.expect(vfs.useDirectIo(slot.file.readyValue().?.*, .auto)); +} + +test "arenas are placed for direct reads from four fifths of the checkpoint's bytes" { + const Case = struct { direct_bytes: u64, bytes: u64, direct: bool }; + const cases = [_]Case{ + .{ .direct_bytes = 0, .bytes = 0, .direct = false }, + .{ .direct_bytes = 4, .bytes = 5, .direct = true }, + .{ .direct_bytes = 79, .bytes = 100, .direct = false }, + .{ .direct_bytes = 80, .bytes = 100, .direct = true }, + .{ .direct_bytes = 0, .bytes = 100, .direct = false }, + }; + for (cases) |case| { + const prediction: Planner.Reads.Prediction = .{ .bytes = case.bytes, .direct_bytes = case.direct_bytes }; + try std.testing.expectEqual(case.direct, prediction.direct()); + } +} + test "a strided part whose source runs are not back to back counts its whole source range" { // [32, 16] u8 at a file offset off a 64-byte alignment. const source: safetensors.Tensor = .{ .file_uri = "wide.bin", .name = "wide", .shape = .init(.{ 32, 16 }, .u8), .offset = 100 }; diff --git a/zml/io/host_memory.zig b/zml/io/host_memory.zig index a8943c36..45063b7e 100644 --- a/zml/io/host_memory.zig +++ b/zml/io/host_memory.zig @@ -10,9 +10,9 @@ //! transfer path are separate decisions: CPU loads through the direct //! backend too. When the task's cpuset and memory policy allow several //! memory-bearing NUMA nodes, page-backed arenas interleave over those of the -//! platform's devices (all of them when a device's node is unknown or none of -//! theirs is allowed); ROCm balances whole arenas across its devices' host -//! nodes. +//! platform's devices (all of them when the load reads past the page cache, +//! a device's node is unknown, or none of theirs is allowed); ROCm balances +//! whole arenas across its devices' host nodes. //! `Workspace.mapped_bytes_ceiling` (16 GiB) is a safety guard, not a //! target. A failed allocation or registration, or a growth past the //! ceiling, is logged with its cause and returns `OutOfMemory`. @@ -77,10 +77,11 @@ const ArenaKind = union(enum) { allocator: Allocator, io: std.Io, platform: *const Platform, + direct_reads: bool, ) Allocator.Error!ArenaKind { return switch (platform.target) { - .cuda, .oneapi => .{ .pages = .{ .platform = platform, .numa_mask = pagesNumaMask(allocator, io, platform) } }, - .cpu => .{ .pages = .{ .platform = null, .numa_mask = pagesNumaMask(allocator, io, platform) } }, + .cuda, .oneapi => .{ .pages = .{ .platform = platform, .numa_mask = pagesNumaMask(allocator, io, platform, direct_reads) } }, + .cpu => .{ .pages = .{ .platform = null, .numa_mask = pagesNumaMask(allocator, io, platform, direct_reads) } }, // Furiosa backs `.host_pinned` with SDK pinned (dma-buf) memory // from `/dev/dma_heap/system`, which the SDK recognizes by // address and transfers without pinning pages on every request. @@ -295,15 +296,19 @@ pub const Workspace = struct { /// The arenas cut into blocks, set by `carve`: nothing maps after it. blocks: ?[]Block = null, + /// `direct_reads`: the load reads its checkpoint past the page cache, so + /// page-backed arenas spread over every candidate node + /// (`placementMask`). pub fn init( allocator: Allocator, io: std.Io, platform: *const Platform, + direct_reads: bool, ) Allocator.Error!Workspace { return .{ .allocator = allocator, .io = io, - .kind = try .init(allocator, io, platform), + .kind = try .init(allocator, io, platform, direct_reads), }; } @@ -441,10 +446,11 @@ fn cpusetMemsMask() u64 { /// The nodes page-backed arenas interleave over (`placementMask`): the /// memory-bearing nodes the task's cpuset and memory policy allow, narrowed -/// to the nodes of the platform's devices. -fn pagesNumaMask(allocator: Allocator, io: std.Io, platform: *const Platform) u64 { +/// to the nodes of the platform's devices, unless the load reads past the +/// page cache. +fn pagesNumaMask(allocator: Allocator, io: std.Io, platform: *const Platform, direct_reads: bool) u64 { const candidates = memoryNodeMask(allocator, io) & cpusetMemsMask() & taskMemoryPolicyMask(); - return placementMask(candidates, deviceNodeMask(platform)); + return placementMask(candidates, deviceNodeMask(platform), direct_reads); } /// The nodes of the platform's devices (`Device.numaNode`), or null when a @@ -461,19 +467,30 @@ fn deviceNodeMask(platform: *const Platform) ?u64 { /// Zero leaves placement to the kernel: with one candidate node or none /// there is nothing to choose, and the task's policy or the only node decides. -/// Among several candidates, the arenas go to the devices' nodes: an arena on -/// another node crosses the inter-socket link on every H2D copy. Two GB300 -/// on one Grace node loaded Qwen3.8-27B-FP8 in 1.265 s with a task policy -/// holding the arenas to that node, against 1.64 s interleaved over both -/// nodes. Devices on several nodes share one pool interleaved over theirs: -/// four GB300 on two nodes loaded replicated DeepSeek-V4-Flash in 4.35 s -/// against 4.92 s with per-node pools. When a device's node is unknown, or -/// none is a candidate, the arenas interleave over every candidate, which no -/// single node beat everywhere. A single node is placed too, since unplaced -/// CUDA arenas follow the registering thread's node; the kernel still falls -/// back when it is full. -fn placementMask(candidates: u64, device_nodes: ?u64) u64 { +/// For reads through the page cache, the arenas go to the devices' nodes +/// among several candidates: an arena on another node crosses the +/// inter-socket link on every H2D copy. Two GB300 on one Grace node loaded +/// Qwen3.8-27B-FP8 in 1.265 s with a task policy holding the arenas to that +/// node, against 1.64 s interleaved over both nodes. Devices on several +/// nodes share one pool interleaved over theirs: four GB300 on two nodes +/// loaded replicated DeepSeek-V4-Flash in 4.35 s against 4.92 s with +/// per-node pools. When a device's node is unknown, or none is a candidate, +/// the arenas interleave over every candidate, which no single node beat +/// everywhere. A single node is placed too, since unplaced CUDA arenas +/// follow the registering thread's node; the kernel still falls back when +/// it is full. +/// A load read past the page cache (`direct_reads`) spreads the arenas over +/// every candidate instead: the drives write into them, and interleaving +/// splits that traffic, and the copies out of them, between the candidates' +/// memory. Over an md RAID0 of four NVMe drives, two on each node, the same +/// two GB300 loaded the same checkpoint forced direct in 1.752 s with the +/// arenas interleaved over both nodes, against 1.873 s with them on the +/// devices' node: staging a read took 0.52 against 0.64 ms, and a 16 MiB +/// read 1.90 against 1.87 ms. The drives' own nodes are not consulted: every +/// candidate matched them on the one host measured. +fn placementMask(candidates: u64, device_nodes: ?u64, direct_reads: bool) u64 { if (@popCount(candidates) < 2) return 0; + if (direct_reads) return candidates; const near = candidates & (device_nodes orelse candidates); return if (near != 0) near else candidates; } @@ -502,20 +519,29 @@ fn elapsedNanoseconds(started: std.Io.Timestamp, finished: std.Io.Timestamp) u64 return @intCast(@max(started.durationTo(finished).nanoseconds, 0)); } -test "automatic NUMA placement keeps the devices' nodes among several candidates" { +test "automatic NUMA placement keeps the devices' nodes for buffered reads and spreads direct reads over every candidate" { // One candidate or none: the kernel places. - try std.testing.expectEqual(@as(u64, 0), placementMask(0, 0b10)); - try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1)); - try std.testing.expectEqual(@as(u64, 0), placementMask(0b1000, null)); + try std.testing.expectEqual(@as(u64, 0), placementMask(0, 0b10, false)); + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1, false)); + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1000, null, false)); // Devices on one of two nodes: that node; on both: both. - try std.testing.expectEqual(@as(u64, 0b10), placementMask(0b11, 0b10)); - try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11)); + try std.testing.expectEqual(@as(u64, 0b10), placementMask(0b11, 0b10, false)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11, false)); // Unknown device nodes, or none among the candidates: every candidate. - try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null)); - try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b100)); + try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null, false)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b100, false)); // A cpuset of nodes 0 and 2 of four, devices on node 1: the cpuset's // nodes, as `pagesNumaMask` narrows the candidates to them first. - try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10)); + try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10, false)); + + // Direct reads: every candidate, wherever the devices are. + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b10, true)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11, true)); + try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null, true)); + // Still nothing to choose with one candidate, and still within the + // cpuset. + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1, true)); + try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10, true)); } test "parseNodeList accepts kernel node lists" { diff --git a/zml/io/loader.zig b/zml/io/loader.zig index 52172959..71c651d3 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -144,9 +144,14 @@ pub const Loader = struct { stats: Stats = .{}, pub const Options = struct { - /// The checkpoint's path or URI, read by `init` only. When `io` is a - /// VFS, the VFS backend serving it sizes the reads - /// (`VFS.readHints`). Null, or a plain `io`: local sizing. + /// The checkpoint's path or URI, read by `init` only, which copies a + /// local one the VFS may read direct for the set-up task + /// (`backend.Options.checkpoint`). When `io` is a VFS, the VFS + /// backend serving it sizes the reads (`VFS.readHints`), and the + /// direct backend decides at set-up how such a local one's files + /// read, and places its pinned memory for that + /// (`direct_loader.Loader.create`). Null, or a plain `io`: local + /// sizing, pinned memory near the devices. checkpoint: ?[]const u8 = null, /// Concurrent source reads, 1 to `max_read_parallelism`, fixed for /// the load. Null: 16 for a local checkpoint, 32 for a high-latency @@ -193,14 +198,18 @@ pub const Loader = struct { /// (`Options.dma_block_size`), a checkpoint whose scheme the VFS does /// not serve, a block calibration may select whose request (the read /// chunk rounded up to whole blocks) passes 32 MiB, or a request too - /// short for a widened direct read is `InvalidOptions`, logged. + /// short for a widened direct read is `InvalidOptions`, logged. A copy + /// of the path of a local checkpoint the VFS may read direct + /// (`Options.checkpoint`) that cannot be allocated is `OutOfMemory`, + /// unlogged. pub fn init( allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options, - ) error{InvalidOptions}!Loader { - const backend_opts = try backendOptions(io, opts); + ) error{ InvalidOptions, OutOfMemory }!Loader { + var backend_opts = try backendOptions(io, opts); + if (backend_opts.checkpoint) |path| backend_opts.checkpoint = try allocator.dupe(u8, path); const setup_args = .{ allocator, io, platform, backend_opts }; return .{ .allocator = allocator, @@ -211,8 +220,11 @@ pub const Loader = struct { }; } - /// `init`'s task: the device allocators warmed, then the backend. + /// `init`'s task: the device allocators warmed, then the backend. It + /// owns `opts.checkpoint`, `init`'s copy, and runs to its end even when + /// `deinit` cancels it. fn setUp(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: backend.Options) anyerror!Backend { + defer if (opts.checkpoint) |path| allocator.free(path); platform.warmupDeviceAllocators(io) catch |err| { load_log.failure(err, "warm the device allocators", .{}); return err; @@ -1157,8 +1169,9 @@ fn checkPlacement(platform: *const Platform, target: Target, from: []const u8) e } /// `opts` as the backends take them, the checkpoint resolved to the read -/// hints of the VFS backend serving it and the read width to its default. -/// Every option is checked here, once. +/// hints of the VFS backend serving it and the read width to its default, +/// and borrowed when it is local and may read direct +/// (`backend.Options.checkpoint`). Every option is checked here, once. fn backendOptions(io: std.Io, opts: Loader.Options) error{InvalidOptions}!backend.Options { const hints: VFS.ReadHints = hints: { const path = opts.checkpoint orelse break :hints .local; @@ -1179,7 +1192,7 @@ fn backendOptions(io: std.Io, opts: Loader.Options) error{InvalidOptions}!backen load_log.err("invalid loader options: checkpoint={?s}, read_chunk_size={d}, expected 1..{d}: InvalidOptions", .{ opts.checkpoint, hints.read_chunk_size, max_request }); return error.InvalidOptions; } - const options: backend.Options = .{ + var options: backend.Options = .{ .read_parallelism = opts.read_parallelism orelse if (hints.high_latency) Loader.Options.high_latency_read_parallelism else @@ -1196,6 +1209,10 @@ fn backendOptions(io: std.Io, opts: Loader.Options) error{InvalidOptions}!backen return error.InvalidOptions; } } + // A nonzero alignment means a VFS that may read a local file direct. + // `isLocal` opens nothing. + const path = opts.checkpoint orelse return options; + if (alignment != 0 and VFS.fromIo(io).?.isLocal(path)) options.checkpoint = path; return options; } @@ -1726,6 +1743,42 @@ test "init checks every option and returns InvalidOptions" { loader.deinit(); } +test "init copies a local checkpoint's path for its set-up" { + if (VFS.direct_io_alignment == 0) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const unit: [VFS.direct_io_alignment]u8 = @splat(7); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors", .data = &unit }); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; + var path_buffer: [std.fs.max_path_bytes + 32]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, "{s}/model.safetensors", .{root}); + + // The copy is the only allocation of `init`: nothing is left to free + // when it fails. + var failing: std.testing.FailingAllocator = .init(allocator, .{ .fail_index = 0 }); + if (Loader.init(failing.allocator(), vfs.io(), platform, .{ .checkpoint = path })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.OutOfMemory, err); + + // The set-up frees `init`'s copy once, whether it ran to its end or + // `deinit` canceled it before any `start`. + var loader = try Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = path }); + defer loader.deinit(); + try loader.awaitSetup(); + for ([_]VFS.DirectIo{ .on, .auto }) |policy| { + var unstarted = try Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = path, .direct_io = policy }); + unstarted.deinit(); + } +} + test "a failed set-up fails the first start, and deinit frees the rest" { const io = std.testing.io; var fixture: LoaderTestFixture = undefined; From 4a68cd606a3d58886bf7e17f411c397ee6c48aee Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 14:49:30 +0100 Subject: [PATCH 091/103] zml/io: export dma_calibration.measure and add load_check's dma-bench dma_calibration.measure runs the direct backend's DMA calibration without a loader: it warms the device allocators, screens the blocks in a workspace of its own, released before it returns, and returns the block it selects. Where the loader only picks the candidate blocks, measure takes every knob of Options and checks them first: an empty block list, a block past the largest source request, a parallelism outside 1..32, a window that is not positive or a tolerance outside [0, 1) is InvalidOptions, logged. A platform that loads through the buffered backend has no DMA to measure and is UnsupportedPlatform. zml.io.next now exports dma_calibration. `load_check dma-bench` is measure's caller. It takes the screen's knobs from ZML_DMA_BENCH_* (candidate blocks in MiB, parallelism, window length, transfer target, tolerance) and the platform's from the same variables as a load, and logs the block selected. load_check also enables the zml/io scope at debug, where the screen logs its pick and the rate it measured on one device. Co-Authored-By: Claude Opus 5.5 --- examples/io/load_check.zig | 68 +++++++++++++++++++++-- zml/io.zig | 4 +- zml/io/dma_calibration.zig | 108 ++++++++++++++++++++++++++++++++++--- 3 files changed, 168 insertions(+), 12 deletions(-) diff --git a/examples/io/load_check.zig b/examples/io/load_check.zig index 9c8f68cd..bdf56f12 100644 --- a/examples/io/load_check.zig +++ b/examples/io/load_check.zig @@ -4,6 +4,7 @@ //! each host. //! //! bazel run //examples/io:load_check -- [replicated|sharded|tp] +//! bazel run //examples/io:load_check -- dma-bench //! //! The checkpoint is a model directory, a .safetensors file or a //! .safetensors.index.json, local or remote (hf://, s3://, gs://, https://, @@ -19,9 +20,26 @@ //! strided in the checkpoint), the first axis of the others. //! A scalar is replicated in every mode. //! +//! `dma-bench` loads nothing: it runs the direct backend's DMA calibration +//! (`zml.io.next.dma_calibration.measure`) on the platform and logs the +//! block it selects. A platform that loads through the buffered backend has +//! no DMA to measure, and CPU selects the smallest block unmeasured. Its +//! knobs, beside the platform's below, are the screen's options, at their +//! defaults when unset: +//! - `ZML_DMA_BENCH_BLOCK_MIB=a,b,...`: the candidate blocks in MiB, each +//! at most 32 (2, 4, 8, 16 and 32); a load takes a single one (below); +//! - `ZML_DMA_BENCH_BLOCK_PARALLELISM=n`: transfers in flight per device, +//! at most 32 (8); +//! - `ZML_DMA_BENCH_WINDOW_MS=n`: a screen window's minimum duration (2); +//! - `ZML_DMA_BENCH_MIN_TRANSFERS=n`: the transfers a window waits for on +//! the measured device, 0 for none (32); +//! - `ZML_DMA_BENCH_BLOCK_TOLERANCE=f`: the smallest block within this +//! fraction of the peak rate is selected (0.08). +//! //! The loader logs the bytes it loaded, its time and rate at info, and its -//! stats at debug, on the `zml/io/load` scope; `zml/vfs` logs the remote -//! read totals. This tool enables both. +//! stats at debug, on the `zml/io/load` scope; `zml/io` logs the DMA +//! calibration's pick with its measured rate on one device (`dma_bench`); +//! `zml/vfs` logs the remote read totals. This tool enables all three. //! //! Knobs, from the environment: //! - `ZML_LOAD_CHECK=n`: once loaded, read every n-th tensor and the largest @@ -54,6 +72,7 @@ pub const std_options: std.Options = .{ .log_level = .info, .log_scope_levels = &.{ .{ .scope = .@"zml/io/load", .level = .debug }, + .{ .scope = .@"zml/io", .level = .debug }, .{ .scope = .@"zml/vfs", .level = .debug }, }, }; @@ -66,9 +85,10 @@ pub fn main(init: std.process.Init) !void { var args = init.minimal.args.iterate(); _ = args.next(); // skip program name const path = args.next() orelse { - log.err("usage: load_check [replicated|sharded|tp]", .{}); + log.err("usage: load_check [replicated|sharded|tp], or load_check dma-bench", .{}); return error.MissingPath; }; + if (std.mem.eql(u8, path, "dma-bench")) return dmaBench(init); const mode = std.meta.stringToEnum(Mode, args.next() orelse "sharded") orelse { log.err("the mode must be replicated, sharded or tp", .{}); return error.InvalidArgument; @@ -147,6 +167,30 @@ pub fn main(init: std.process.Init) !void { if (check_stride != 0) try checkLoaded(allocator, io, sources, buffers.tensors, check_stride); } +/// Runs `dma-bench`: the DMA calibration screen on the platform, with the +/// `ZML_DMA_BENCH_*` knobs. +fn dmaBench(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const environ_map = init.environ_map; + + const defaults: zml.io.next.dma_calibration.Options = .{}; + const window_ms = try envNumber(i64, environ_map, "ZML_DMA_BENCH_WINDOW_MS"); + const opts: zml.io.next.dma_calibration.Options = .{ + .block_sizes = try envMibList(init.arena.allocator(), environ_map, "ZML_DMA_BENCH_BLOCK_MIB") orelse defaults.block_sizes, + .block_parallelism = try envNumber(usize, environ_map, "ZML_DMA_BENCH_BLOCK_PARALLELISM") orelse defaults.block_parallelism, + .minimum_duration = if (window_ms) |ms| .fromMilliseconds(ms) else defaults.minimum_duration, + .minimum_transfers = try envNumber(u64, environ_map, "ZML_DMA_BENCH_MIN_TRANSFERS") orelse defaults.minimum_transfers, + .block_selection_tolerance = try envNumber(f64, environ_map, "ZML_DMA_BENCH_BLOCK_TOLERANCE") orelse defaults.block_selection_tolerance, + }; + + const platform: *zml.Platform = try .auto(allocator, io, try platformOptions(environ_map)); + defer platform.deinit(allocator, io); + + const block_size = try zml.io.next.dma_calibration.measure(allocator, io, platform, opts); + log.info("dma_benchmark block_bytes={d} parallelism={d}", .{ block_size, opts.block_parallelism }); +} + /// The platform settings of the environment, which are not loader knobs. fn platformOptions(environ_map: *const std.process.Environ.Map) !zml.platform.CreateOptions { var options: zml.platform.CreateOptions = .{}; @@ -271,6 +315,24 @@ fn expectSameBytes(name: []const u8, replica: ?usize, expected: []const u8, actu return error.LoadContentMismatch; } +/// The environment variable `name` as a comma-separated list of MiB, in +/// bytes, null when unset. An entry that does not parse is +/// `InvalidArgument`, logged. +fn envMibList(allocator: std.mem.Allocator, environ_map: *const std.process.Environ.Map, name: []const u8) !?[]const usize { + const text = environ_map.get(name) orelse return null; + var sizes: std.ArrayList(usize) = .empty; + errdefer sizes.deinit(allocator); + var entries = std.mem.tokenizeScalar(u8, text, ','); + while (entries.next()) |entry| { + const mib = std.fmt.parseInt(u32, entry, 10) catch { + log.err("{s} must be MiB counts separated by commas, not '{s}'", .{ name, text }); + return error.InvalidArgument; + }; + try sizes.append(allocator, @as(usize, mib) * zml.MiB); + } + return try sizes.toOwnedSlice(allocator); +} + /// The environment variable `name` parsed as a `T`, null when unset. A value /// that does not parse is `InvalidArgument`, logged. fn envNumber(comptime T: type, environ_map: *const std.process.Environ.Map, name: []const u8) !?T { diff --git a/zml/io.zig b/zml/io.zig index 3a633c65..44d52be0 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -25,11 +25,13 @@ const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/io"); /// A preview of the loader that replaces `Loader` and `TensorStore` below: -/// the new `Loader` (io/loader.zig) and its store (io/TensorStore.zig). +/// the new `Loader` (io/loader.zig), its store (io/TensorStore.zig) and +/// the DMA calibration of its direct backend (io/dma_calibration.zig). /// Unstable: their API may still change until they replace the old ones. pub const next = struct { pub const Loader = @import("io/loader.zig").Loader; pub const TensorStore = @import("io/TensorStore.zig"); + pub const dma_calibration = @import("io/dma_calibration.zig"); }; // Every file of the new loader under io/, for its tests: `next` exports diff --git a/zml/io/dma_calibration.zig b/zml/io/dma_calibration.zig index dd4e0e30..ecb88e7c 100644 --- a/zml/io/dma_calibration.zig +++ b/zml/io/dma_calibration.zig @@ -8,12 +8,16 @@ //! CPU returns the smallest candidate without measuring: its plugin's //! transfer is a memcpy, and a load takes the same time at every block //! size. The buffered backend has nothing to size. +//! +//! `measure` runs the same warm-up and screen without a loader, with every +//! knob, for load_check's `dma-bench` (examples/io/load_check.zig). const std = @import("std"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Platform = @import("../platform.zig").Platform; +const backend = @import("backend.zig"); const host_memory = @import("host_memory.zig"); const log = @import("log.zig").io; @@ -37,11 +41,11 @@ pub const default_block_sizes = [_]usize{ }; /// The screen's knobs. The loader sets only `block_sizes` and keeps the -/// other defaults. +/// other defaults; `measure` takes them all. pub const Options = struct { block_sizes: []const usize = &default_block_sizes, - /// Transfers in flight per device in the screen. The loader keeps - /// `depth`. + /// Transfers in flight per device in the screen, at most + /// `max_block_parallelism`. The loader keeps `depth`. block_parallelism: usize = depth, /// A screen window runs for at least this long and, unless the target is /// zero, until the representative device completes the transfer target. @@ -64,14 +68,75 @@ const confirmation_duration: std.Io.Duration = .fromMilliseconds(25); const confirmation_minimum_transfers: u64 = 256; const confirmation_margin: f64 = 0.02; +/// The widest screen `measure` takes. +pub const max_block_parallelism: usize = 32; + +/// Measures as the direct loader's creation does, without a loader and with +/// every knob of `opts`, and returns the block it selects: warms the device +/// allocators, then screens the blocks in a workspace of its own, placed +/// near the devices as for reads through the page cache (it reads no file), +/// released before it returns. CPU returns the smallest block without +/// measuring; a platform that loads through the buffered backend +/// (`backend.loadsDirect`) returns `UnsupportedPlatform`, and a knob out of +/// range `InvalidOptions`. Every failure is logged. For load_check's +/// `dma-bench`. +pub fn measure(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options) !usize { + try validate(opts); + if (!backend.loadsDirect(platform)) { + log.err("DMA calibration: {t} loads through the buffered backend, which has no DMA to measure: UnsupportedPlatform", .{platform.target}); + return error.UnsupportedPlatform; + } + platform.warmupDeviceAllocators(io) catch |err| { + log.failure(err, "DMA calibration: warm the device allocators: target={t}", .{platform.target}); + return err; + }; + var workspace = try host_memory.Workspace.init(allocator, io, platform, false); + defer workspace.deinit(); + return calibrate(&workspace, platform, opts) catch |err| { + log.failure(err, "DMA calibration: target={t}", .{platform.target}); + return err; + }; +} + +/// A block is at most the largest source request, the bound +/// `zml.io.next.Loader.init` puts on its own block sizes, so every +/// candidate's ring of `block_parallelism` blocks fits the workspace's +/// mapped ceiling. +fn validate(opts: Options) error{InvalidOptions}!void { + comptime std.debug.assert(backend.max_read_request_size * max_block_parallelism <= host_memory.Workspace.mapped_bytes_ceiling); + if (opts.block_parallelism == 0 or opts.block_parallelism > max_block_parallelism) { + log.err("invalid DMA calibration options: block_parallelism={d}, expected 1..{d}: InvalidOptions", .{ opts.block_parallelism, max_block_parallelism }); + return error.InvalidOptions; + } + if (opts.block_sizes.len == 0) { + log.err("invalid DMA calibration options: block_sizes is empty: InvalidOptions", .{}); + return error.InvalidOptions; + } + for (opts.block_sizes) |block_size| { + if (block_size == 0 or block_size > backend.max_read_request_size) { + log.err("invalid DMA calibration options: block size {d}, expected 1..{d}: InvalidOptions", .{ block_size, backend.max_read_request_size }); + return error.InvalidOptions; + } + } + if (opts.minimum_duration.nanoseconds <= 0) { + log.err("invalid DMA calibration options: minimum_duration={f}, expected more than zero: InvalidOptions", .{opts.minimum_duration}); + return error.InvalidOptions; + } + // Written so that NaN fails too. + if (!(opts.block_selection_tolerance >= 0 and opts.block_selection_tolerance < 1)) { + log.err("invalid DMA calibration options: block_selection_tolerance={d}, expected [0, 1): InvalidOptions", .{opts.block_selection_tolerance}); + return error.InvalidOptions; + } +} + /// Measures one representative device into the direct loader's workspace /// and returns the block it selects; CPU returns the smallest without /// measuring. On a platform mixing device kinds, device 0's block serves -/// every device: it is a tuning choice, valid for all of them. Each block -/// of `opts` is nonzero and at most `backend.max_read_request_size`, so the -/// ring fits the workspace's mapped ceiling. The rate the `dma_bench` line -/// logs is that one device's synthetic H2D rate, not the platform's nor a -/// prediction of a checkpoint's load speed. +/// every device: it is a tuning choice, valid for all of them. `opts` were +/// checked (`validate`, or `zml.io.next.Loader.init` for the loader's block +/// sizes). The rate the `dma_bench` line logs is that one device's +/// synthetic H2D rate, not the platform's nor a prediction of a +/// checkpoint's load speed. pub fn calibrate( workspace: *host_memory.Workspace, platform: *const Platform, @@ -533,6 +598,33 @@ test "DMA benchmark confirms a candidate when round qualification disagrees" { try std.testing.expect(isBorderline(&candidates, 0, 0.80, 0.05)); } +test "measure refuses a knob out of range before it touches the platform" { + const cases = [_]Options{ + .{ .block_sizes = &.{} }, + .{ .block_sizes = &.{0} }, + .{ .block_sizes = &.{backend.max_read_request_size + 1} }, + .{ .block_parallelism = 0 }, + .{ .block_parallelism = max_block_parallelism + 1 }, + .{ .minimum_duration = .zero }, + .{ .minimum_duration = .fromMilliseconds(-1) }, + .{ .block_selection_tolerance = 1 }, + .{ .block_selection_tolerance = -0.01 }, + .{ .block_selection_tolerance = std.math.nan(f64) }, + }; + for (cases) |opts| { + try std.testing.expectError(error.InvalidOptions, measure(std.testing.allocator, std.testing.io, undefined, opts)); + } +} + +test "measure needs no loader and selects the smallest block on CPU" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + try std.testing.expectEqual(4 << 20, try measure(allocator, io, platform, .{ .block_sizes = &.{ 8 << 20, 4 << 20 } })); +} + test "DMA benchmark shares one manager across candidate sizes and slots" { const allocator = std.testing.allocator; const io = std.testing.io; From 6cf8cc3d7827a10751d9671cfad5edd2648947a2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 17:01:17 +0100 Subject: [PATCH 092/103] zml/io: test a rate-limited HTTP load end to end A loader test serves four tensors from a MockServer that lets two GETs through per 100 ms and answers the others with 429, and loads them over the VFS's HTTP backend with four readers. Every tensor must land intact, the governor must have throttled and held the readers without spending a retry, and the physical bytes must equal the tensors'. The direct backend must size its requests from the HTTP backend's hints rather than the local ones and read the remote file exact, without direct I/O. Its destroy, which deinit runs, asserts that every lifecycle credit came back once the workers stopped; the test does not read the credits before, while a worker may still hold one for a claim that finds nothing. Co-Authored-By: Claude Opus 5.5 --- zml/io/loader.zig | 95 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/zml/io/loader.zig b/zml/io/loader.zig index 71c651d3..14c10072 100644 --- a/zml/io/loader.zig +++ b/zml/io/loader.zig @@ -1832,6 +1832,101 @@ test "without concurrency init sets up in place" { try testing.expectShards(allocator, io, &output, &LoaderTestFixture.contents, .default); } +test "a rate-limited HTTP source loads through the VFS hold" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const tensor_bytes = 1024; + const tensors = 4; + + // Four tensor ranges with a gap between them, so the planner keeps four + // jobs and the load makes four concurrent GETs. + var object: [(2 * tensors - 1) * tensor_bytes]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 31 +% 7); + + var server = try VFS.MockServer.init(io, &object, .{ + // Two GETs per 100 ms: the four readers are throttled repeatedly. + .throttle = .{ .window = .{ .gets = 2, .per_ms = 100 } }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var http: VFS.HTTP = try .init(allocator, io, &client, .{ .protocol = .http, .retry = .{ + .max_retries = 5, + .initial_delay = .fromMilliseconds(20), + .max_delay = .fromMilliseconds(20), + .max_hold = .fromSeconds(5), + .throttle_budget = .fromSeconds(30), + } }); + defer http.deinit(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + try vfs.registerBackend("http", http.backend()); + const stats = &http.governor.stats; + + var url_buffer: [128]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + + const platform = testing.env(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + const names = [tensors][]const u8{ "a", "b", "c", "d" }; + for (names, 0..) |name, index| { + try registry.registerTensor(.{ + .file_uri = url, + .name = name, + .shape = .init(.{tensor_bytes}, .u8), + .offset = 2 * index * tensor_bytes, + }); + } + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + + const Model = struct { a: Tensor, b: Tensor, c: Tensor, d: Tensor }; + const model: Model = .{ + .a = store.view().createReplicatedTensor("a", null), + .b = store.view().createReplicatedTensor("b", null), + .c = store.view().createReplicatedTensor("c", null), + .d = store.view().createReplicatedTensor("d", null), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + + var loader = try Loader.init(allocator, vfs.io(), platform, .{ + .checkpoint = url, + .read_parallelism = 4, + }); + // Holding costs time, not credits: the direct backend's `destroy` + // asserts that every lifecycle credit came back with its pinned blocks. + // It checks once its workers stopped, since until then one may hold a + // credit for a claim that finds nothing. + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + inline for (comptime std.meta.fieldNames(Model), 0..) |field_name, index| { + try testing.expectShards(allocator, io, &@field(buffers, field_name), object[2 * index * tensor_bytes ..][0..tensor_bytes], .default); + } + + try server.stop(io); + + // The server rate limited the load, the backend held every reader, and + // no reader burned its retry budget for it. + try std.testing.expect(stats.throttles.load(.acquire) > 0); + try std.testing.expect(stats.holds.load(.acquire) > 0); + try std.testing.expect(stats.hold_wait_ns.load(.acquire) > 0); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, tensors * tensor_bytes), stats.physical_bytes.load(.acquire)); + + // The checkpoint's VFS backend, not the local hints (8 MiB), sized the + // reads: 16 MiB requests over the CPU's 2 MiB blocks. Its file was read + // exact: a remote file is never direct. + const direct = loader.backend.?.direct; + try std.testing.expectEqual(@as(usize, 16 << 20), direct.plan_config.request_size); + try std.testing.expectEqual(@as(usize, 0), direct.source_slots.get(url).?.alignment orelse 0); +} + test "a rejected binding or a program of the wrong output fails the loader" { const allocator = std.testing.allocator; const io = std.testing.io; From 5b0c5fc466d33ac616d8195ccae2cd587a8e9a43 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 18:57:52 +0100 Subject: [PATCH 093/103] zml/io: export the new Loader and TensorStore from zml.io; keep next as an alias zml.io.Loader and zml.io.TensorStore are now the loader and store of the io/ tree, with dma_calibration beside them, and io.zig opens with a map of that tree's files. zml.io.next stays as a second name for the module, so code written against it builds unchanged. The io tree's docs and messages name zml.io. CPU now loads through the direct backend, as CUDA, ROCm and oneAPI do. On 4 CPU devices, Llama-3.1-8B-Instruct sharded by rows loads in about 1.60 s, against 1.78 s through the buffered backend and 9.6 s through the previous loader. The previous store and loader stay in io.zig under private names, with their tests, until the following commits move their callers and delete them; the examples build again once they move. Co-Authored-By: Claude Opus 5.5 --- zml/io.zig | 104 ++++++++++++++++++++----------------- zml/io/TensorStore.zig | 7 +-- zml/io/backend.zig | 8 +-- zml/io/direct_loader.zig | 6 +-- zml/io/dma_calibration.zig | 6 +-- 5 files changed, 68 insertions(+), 63 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index 44d52be0..e76f47ae 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1,3 +1,23 @@ +//! Checkpoint loading and virtual filesystem integration. Each file's +//! header holds its internals: +//! +//! - io/loader.zig: options and their checks, the set-up task, starts, +//! program compiles and groups, admission, and the one check of every +//! binding before the backends see it (`LoadSpec`). +//! - io/backend.zig: backend selection, the request size rule and the +//! `LoadSpec` contract. +//! - io/TensorStore.zig, io/Source.zig: bindings, views, and the sources +//! that describe a tensor's checkpoint bytes. +//! - io/direct_loader.zig: sizing, per-file planning, FIFO claims, source +//! workers, per-device pumps and transfer completion. +//! - io/ShardGrid.zig: where a destination's logical bytes land. +//! - io/buffered_loader.zig: whole-tensor staging and positional reads. +//! - io/dma_calibration.zig: DMA block selection. +//! - io/host_memory.zig: pinned host arenas and their blocks. +//! - io/log.zig: log scopes. Loader test support is in `zml.testing`. +//! - vfs: local and remote files behind one `std.Io`, the remote error +//! contract, retries and the rate-limit hold (vfs/request.zig). + const std = @import("std"); const pjrt = @import("pjrt"); @@ -24,32 +44,20 @@ const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/io"); -/// A preview of the loader that replaces `Loader` and `TensorStore` below: -/// the new `Loader` (io/loader.zig), its store (io/TensorStore.zig) and -/// the DMA calibration of its direct backend (io/dma_calibration.zig). -/// Unstable: their API may still change until they replace the old ones. -pub const next = struct { - pub const Loader = @import("io/loader.zig").Loader; - pub const TensorStore = @import("io/TensorStore.zig"); - pub const dma_calibration = @import("io/dma_calibration.zig"); -}; +pub const Loader = @import("io/loader.zig").Loader; +pub const TensorStore = @import("io/TensorStore.zig"); + +pub const dma_calibration = @import("io/dma_calibration.zig"); + +/// This module under a second name, for code written against it: +/// `zml.io.next.Loader` is `zml.io.Loader`. +pub const next = @This(); -// Every file of the new loader under io/, for its tests: `next` exports -// only some of them. test { - _ = @import("io/ShardGrid.zig"); - _ = @import("io/Source.zig"); - _ = @import("io/TensorStore.zig"); - _ = @import("io/backend.zig"); - _ = @import("io/buffered_loader.zig"); - _ = @import("io/direct_loader.zig"); - _ = @import("io/dma_calibration.zig"); - _ = @import("io/host_memory.zig"); - _ = @import("io/loader.zig"); - _ = @import("io/log.zig"); + std.testing.refAllDecls(@This()); } -pub const TensorStore = struct { +const OldTensorStore = struct { pub const Binding = struct { tensors: []*safetensors.Tensor, transformed: bool, @@ -76,7 +84,7 @@ pub const TensorStore = struct { /// A store over `registry`, which it borrows: the registry must outlive /// the store, and may still gain entries, which views then find. Each /// tensor the store binds holds a copy of its entry. - pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, opts: Options) TensorStore { + pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, opts: Options) OldTensorStore { var arena: std.heap.ArenaAllocator = .init(allocator); return .{ .registry = registry, @@ -88,12 +96,12 @@ pub const TensorStore = struct { }; } - pub fn deinit(self: *TensorStore) void { + pub fn deinit(self: *OldTensorStore) void { self.id_to_sources.deinit(self.allocator); self.arena.deinit(); } - fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { + fn putSourcesNoClobber(self: *OldTensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { const gop = try self.id_to_sources.getOrPut(self.allocator, id); if (gop.found_existing) { stdx.debug.panic("Id {} already has associated sources", .{id}); @@ -103,12 +111,12 @@ pub const TensorStore = struct { gop.value_ptr.* = sources; } - fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*safetensors.Tensor { + fn getPtrFromKey(self: *const OldTensorStore, key: []const u8) ?*safetensors.Tensor { const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; return tensor_desc_ptr; } - fn dupeSource(self: *TensorStore, key: []const u8) ?*safetensors.Tensor { + fn dupeSource(self: *OldTensorStore, key: []const u8) ?*safetensors.Tensor { const entry = self.getPtrFromKey(key) orelse return null; const copy = self.arena.allocator().create(safetensors.Tensor) catch @panic("OOM"); @@ -117,32 +125,32 @@ pub const TensorStore = struct { return copy; } - fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { + fn getPtrFromId(self: *const OldTensorStore, id: usize) ?*safetensors.Tensor { const sources = self.id_to_sources.get(id) orelse return null; stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); return sources.tensors[0]; } - pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { + pub fn getReader(self: *const OldTensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { return self.registry.reader(io, key, buffer); } - pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { + pub fn getReaderById(self: *const OldTensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { const sources = self.id_to_sources.get(id) orelse return error.NotFound; stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); return sources.tensors[0].reader(io, buffer, .{}); } - pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?Binding { + pub fn getSourcesById(self: *const OldTensorStore, id: Tensor.Id) ?Binding { return self.id_to_sources.get(id); } - pub fn view(self: *TensorStore) View { + pub fn view(self: *OldTensorStore) View { return .{ .store = self }; } - pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + pub fn mesh(store: *const OldTensorStore, name: @EnumLiteral()) *const Sharding.Mesh { const name_slice = @tagName(name); for (store.meshes) |m| { if (std.mem.eql(u8, name_slice, m.name)) { @@ -157,7 +165,7 @@ pub const TensorStore = struct { } pub const View = struct { - store: *TensorStore, + store: *OldTensorStore, prefix_buffer: [256]u8 = undefined, prefix_length: usize = 0, @@ -340,7 +348,7 @@ pub const TensorStore = struct { }; }; -pub const Loader = struct { +const OldLoader = struct { allocator: std.mem.Allocator, platform: *const Platform, dma_allocators: []const mem.DmaAllocator, @@ -361,7 +369,7 @@ pub const Loader = struct { dma_chunk_size: usize, }; - pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!Loader { + pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!OldLoader { const pool_count = platform.devices.len; const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); errdefer allocator.free(dma_allocators); @@ -388,14 +396,14 @@ pub const Loader = struct { }; } - pub fn deinit(self: *Loader) void { + pub fn deinit(self: *OldLoader) void { self.delivered.deinit(self.allocator); for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); self.allocator.free(self.pinned_buffer_pools); self.allocator.free(self.dma_allocators); } - pub fn await(self: *Loader, io: std.Io) std.Io.Cancelable!void { + pub fn await(self: *OldLoader, io: std.Io) std.Io.Cancelable!void { return self.group.await(io); } @@ -405,7 +413,7 @@ pub const Loader = struct { pub const LoadError = error{TransformedTensorNotDelivered}; - pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { + pub fn load(self: *OldLoader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const OldTensorStore, opts: LoadOpts) LoadError!void { const tensor_count = meta.count(Tensor, model); var arena: std.heap.ArenaAllocator = .init(self.allocator); @@ -464,11 +472,11 @@ pub const Loader = struct { log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); } - pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { + pub fn loadBuffer(loader: *OldLoader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const OldTensorStore, opts: LoadOpts) void { loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); } - fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { + fn defaultCallback(self: *OldLoader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const OldTensorStore, opts: LoadOpts) void { const sources = store.getSourcesById(tensor.id) orelse { std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); return; @@ -481,7 +489,7 @@ pub const Loader = struct { }; } - fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { + fn loadSingle(self: *OldLoader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { log.err("Failed to load tensor {s}: {}", .{ source.name, e }); loaded.* = false; @@ -491,7 +499,7 @@ pub const Loader = struct { } fn loadSingleInner( - self: *Loader, + self: *OldLoader, io: std.Io, source: *safetensors.Tensor, shape: Shape, @@ -534,12 +542,12 @@ pub const Loader = struct { } pub fn loadExecute( - self: *Loader, + self: *OldLoader, arena: std.mem.Allocator, io: std.Io, tensor: Tensor, buffer: *Buffer, - store: *const TensorStore, + store: *const OldTensorStore, exe: *const Exe, opts: LoadOpts, ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { @@ -593,7 +601,7 @@ test "View.get finds exact keys in the registry the store borrows" { try @import("testing.zig").writeSafetensors(allocator, io, tmp.dir, "layer.safetensors", ®istry, &.{ .{ .name = "layer.experts.0.down_proj.weight", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, }); - var store: TensorStore = .init(allocator, ®istry, .{ .quantization = .{ .fp8 = .f8e4m3fnuz } }); + var store: OldTensorStore = .init(allocator, ®istry, .{ .quantization = .{ .fp8 = .f8e4m3fnuz } }); defer store.deinit(); const layer = store.view().withPrefix("layer"); @@ -631,9 +639,9 @@ test "Loader.loadExecute cleans up donated and non-donated inputs" { defer registry.deinit(); try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); + var store: OldTensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); defer store.deinit(); - var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); + var loader = try OldLoader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); defer loader.deinit(); inline for (.{ false, true }) |donate| { diff --git a/zml/io/TensorStore.zig b/zml/io/TensorStore.zig index 78567334..3b159bbc 100644 --- a/zml/io/TensorStore.zig +++ b/zml/io/TensorStore.zig @@ -1,8 +1,5 @@ //! Associates model tensors with checkpoint sources and prefixed views, -//! over the entries it takes from a checkpoint registry. zml exports it as -//! `zml.io.next.TensorStore`, the store `zml.io.next.Loader` loads from; -//! `zml.io.TensorStore` (io.zig) is the old loader's, which borrows its -//! registry. +//! over the entries it takes from a checkpoint registry. //! //! The store owns those entries, the bindings, and every source and program //! value (its arena). Views and sources point to it: once `view` was @@ -285,7 +282,7 @@ pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mes } std.debug.panic( \\Found no meshes named {s} in TensorStore. - \\Try passing more meshes to `zml.io.next.TensorStore.init`. + \\Try passing more meshes to `zml.io.TensorStore.init`. \\Known meshes: {f} , .{ name_slice, stdx.fmt.slice(store.meshes) }); } diff --git a/zml/io/backend.zig b/zml/io/backend.zig index 722c4e0a..f118f5fe 100644 --- a/zml/io/backend.zig +++ b/zml/io/backend.zig @@ -25,7 +25,7 @@ pub const max_read_request_size: usize = 32 * 1024 * 1024; /// a nonzero multiple of `alignment` (`readAlignment`: reads land in /// blocks), and the request is at most `max_read_request_size` and, with an /// alignment, longer than two units, so a read widened by a unit at each -/// end fits. `zml.io.next.Loader.init` refuses options for which a block +/// end fits. `zml.io.Loader.init` refuses options for which a block /// calibration may select has none. pub fn requestSize(read_chunk_size: usize, block_size: usize, alignment: usize) ?usize { if (block_size == 0 or block_size % @max(alignment, 1) != 0) return null; @@ -47,9 +47,9 @@ pub fn readAlignment(io: std.Io, direct_io: VFS.DirectIo) usize { return VFS.direct_io_alignment; } -/// `zml.io.next.Loader.Options` as `zml.io.next.Loader.init` checked them, -/// the checkpoint resolved to its VFS backend's read hints and the read -/// width to its default. See `zml.io.next.Loader.Options` for each field. +/// `zml.io.Loader.Options` as `zml.io.Loader.init` checked them, the +/// checkpoint resolved to its VFS backend's read hints and the read width to +/// its default. See `zml.io.Loader.Options` for each field. pub const Options = struct { read_parallelism: usize, read_hints: VFS.ReadHints = .local, diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig index 28a69375..a68e883b 100644 --- a/zml/io/direct_loader.zig +++ b/zml/io/direct_loader.zig @@ -215,9 +215,9 @@ pub const Loader = struct { } var workspace: host_memory.Workspace = try .init(allocator, io, platform, predicted.direct()); errdefer workspace.deinit(); - // At debug when canceled (`zml.io.next.Loader.deinit` cancels a - // set-up no `start` waited for) or out of memory (the workspace logs - // what it fails to map or pin). + // At debug when canceled (`zml.io.Loader.deinit` cancels a set-up + // no `start` waited for) or out of memory (the workspace logs what + // it fails to map or pin). const block_size = dma_calibration.calibrate(&workspace, platform, .{ .block_sizes = opts.dmaBlockSizes() }) catch |err| { load_log.failure(err, "calibrate DMA: target={t}", .{platform.target}); return err; diff --git a/zml/io/dma_calibration.zig b/zml/io/dma_calibration.zig index ecb88e7c..e744a9e6 100644 --- a/zml/io/dma_calibration.zig +++ b/zml/io/dma_calibration.zig @@ -1,6 +1,6 @@ //! DMA sizing for the direct backend: the block size, decided once when the //! backend is created. It screens `default_block_sizes` (2 to 32 MiB), or -//! the one block `zml.io.next.Loader.Options.dma_block_size` gives, on one +//! the one block `zml.io.Loader.Options.dma_block_size` gives, on one //! representative device, at `depth` transfers per device and over fixed //! windows, and keeps the smallest block within 8% of the measured peak; //! borderline candidates get longer, alternating confirmation windows. The @@ -99,7 +99,7 @@ pub fn measure(allocator: std.mem.Allocator, io: std.Io, platform: *const Platfo } /// A block is at most the largest source request, the bound -/// `zml.io.next.Loader.init` puts on its own block sizes, so every +/// `zml.io.Loader.init` puts on its own block sizes, so every /// candidate's ring of `block_parallelism` blocks fits the workspace's /// mapped ceiling. fn validate(opts: Options) error{InvalidOptions}!void { @@ -133,7 +133,7 @@ fn validate(opts: Options) error{InvalidOptions}!void { /// and returns the block it selects; CPU returns the smallest without /// measuring. On a platform mixing device kinds, device 0's block serves /// every device: it is a tuning choice, valid for all of them. `opts` were -/// checked (`validate`, or `zml.io.next.Loader.init` for the loader's block +/// checked (`validate`, or `zml.io.Loader.init` for the loader's block /// sizes). The rate the `dma_bench` line logs is that one device's /// synthetic H2D rate, not the platform's nor a prediction of a /// checkpoint's load speed. From 2b9119822df6b71aaff4ac42a9cf2b277daa5b5d Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 19:00:19 +0100 Subject: [PATCH 094/103] examples: load with zml.io.Loader The examples load through zml.io.Loader: bufferize the model, start, wait, and free the buffers with deinitBufferized. The loader logs the bytes, time and rate itself, so the hand-written timers go. examples/llm creates its loader in a block that initializes, compiles and loads the model: the loader sets its backend up while the model compiles, and the block frees it, with its pinned host memory, before the chat starts. The weights still load after compilation, to leave room for autotuning. The lfm2 and llama tests take a loader with default options and pass their live progress node. mnist and mnist_dp read their checkpoint through a VFS, create the loader before compiling, and load in main, without a Mnist.load or unloadBuffers. examples/io's load is the same recipe over every checkpoint tensor, and load_check names zml.io. Co-Authored-By: Claude Opus 5.5 --- examples/io/BUILD.bazel | 2 +- examples/io/load_check.zig | 20 ++++----- examples/io/main.zig | 51 +++++++-------------- examples/llm/main.zig | 26 ++++++++--- examples/llm/models.zig | 40 +++++------------ examples/llm/models/lfm2_tests.zig | 6 +-- examples/llm/models/llama_tests.zig | 6 +-- examples/mnist/mnist.zig | 61 ++++++++------------------ examples/mnist/mnist_data_parallel.zig | 49 +++++++-------------- 9 files changed, 98 insertions(+), 163 deletions(-) diff --git a/examples/io/BUILD.bazel b/examples/io/BUILD.bazel index 785ab603..adc3882b 100644 --- a/examples/io/BUILD.bazel +++ b/examples/io/BUILD.bazel @@ -10,7 +10,7 @@ zig_binary( visibility = ["//visibility:public"], ) -# Tooling: the new loader's validation harness. Its header lists its knobs. +# Tooling: the loader's validation harness. Its header lists its knobs. zig_binary( name = "load_check", main = "load_check.zig", diff --git a/examples/io/load_check.zig b/examples/io/load_check.zig index bdf56f12..ad98ce6a 100644 --- a/examples/io/load_check.zig +++ b/examples/io/load_check.zig @@ -1,7 +1,7 @@ -//! Loads every tensor of a checkpoint through `zml.io.next` and checks the +//! Loads every tensor of a checkpoint with `zml.io.Loader` and checks the //! bytes the devices hold against the checkpoint's. Tooling, not an -//! example: the harness the new loader's backends are validated with on -//! each host. +//! example: the harness the loader's backends are validated with on each +//! host. //! //! bazel run //examples/io:load_check -- [replicated|sharded|tp] //! bazel run //examples/io:load_check -- dma-bench @@ -21,7 +21,7 @@ //! A scalar is replicated in every mode. //! //! `dma-bench` loads nothing: it runs the direct backend's DMA calibration -//! (`zml.io.next.dma_calibration.measure`) on the platform and logs the +//! (`zml.io.dma_calibration.measure`) on the platform and logs the //! block it selects. A platform that loads through the buffered backend has //! no DMA to measure, and CPU selects the smallest block unmeasured. Its //! knobs, beside the platform's below, are the screen's options, at their @@ -99,7 +99,7 @@ pub fn main(init: std.process.Init) !void { return error.InvalidArgument; }; const dma_block_mib = try envNumber(u32, init.environ_map, "ZML_DMA_BENCH_BLOCK_MIB"); - const memory = std.meta.stringToEnum(zml.io.next.TensorStore.Memory, init.environ_map.get("ZML_LOAD_MEMORY") orelse "default") orelse { + const memory = std.meta.stringToEnum(zml.io.TensorStore.Memory, init.environ_map.get("ZML_LOAD_MEMORY") orelse "default") orelse { log.err("ZML_LOAD_MEMORY must be default, host_pinned or host_mapped", .{}); return error.InvalidArgument; }; @@ -118,7 +118,7 @@ pub fn main(init: std.process.Init) !void { // Returns at once: the loader sets its backend up while the // checkpoint's headers are read and its tensors bound. - var loader: zml.io.next.Loader = try .init(allocator, io, platform, .{ + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path, .read_parallelism = try envNumber(usize, init.environ_map, "ZML_LOAD_READ_PARALLELISM"), .dma_block_size = if (dma_block_mib) |mib| @as(usize, mib) * zml.MiB else null, @@ -134,7 +134,7 @@ pub fn main(init: std.process.Init) !void { const rows: zml.Sharding.Partitioning = .init(&.{.sharded(model_axis)}); const columns: zml.Sharding.Partitioning = .init(&.{ .replicated, .sharded(model_axis) }); - var store: zml.io.next.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); defer store.deinit(); const Model = struct { tensors: []zml.Tensor }; @@ -174,9 +174,9 @@ fn dmaBench(init: std.process.Init) !void { const io = init.io; const environ_map = init.environ_map; - const defaults: zml.io.next.dma_calibration.Options = .{}; + const defaults: zml.io.dma_calibration.Options = .{}; const window_ms = try envNumber(i64, environ_map, "ZML_DMA_BENCH_WINDOW_MS"); - const opts: zml.io.next.dma_calibration.Options = .{ + const opts: zml.io.dma_calibration.Options = .{ .block_sizes = try envMibList(init.arena.allocator(), environ_map, "ZML_DMA_BENCH_BLOCK_MIB") orelse defaults.block_sizes, .block_parallelism = try envNumber(usize, environ_map, "ZML_DMA_BENCH_BLOCK_PARALLELISM") orelse defaults.block_parallelism, .minimum_duration = if (window_ms) |ms| .fromMilliseconds(ms) else defaults.minimum_duration, @@ -187,7 +187,7 @@ fn dmaBench(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, try platformOptions(environ_map)); defer platform.deinit(allocator, io); - const block_size = try zml.io.next.dma_calibration.measure(allocator, io, platform, opts); + const block_size = try zml.io.dma_calibration.measure(allocator, io, platform, opts); log.info("dma_benchmark block_bytes={d} parallelism={d}", .{ block_size, opts.block_parallelism }); } diff --git a/examples/io/main.zig b/examples/io/main.zig index 03e708b6..e6132ff8 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -3,8 +3,6 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; -const log = std.log.scoped(.vfs); - pub const std_options: std.Options = .{ .log_level = .info, }; @@ -140,6 +138,11 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); + // Returns at once: the loader sets its backend up while the + // checkpoint's headers are read and its tensors bound. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); + defer loader.deinit(); + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); defer registry.deinit(); @@ -149,6 +152,7 @@ pub fn main(init: std.process.Init) !void { ); const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); + // The store takes the registry's entries. var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); defer store.deinit(); @@ -156,51 +160,28 @@ pub fn main(init: std.process.Init) !void { tensors: []zml.Tensor, }; - const tensor_count = registry.tensors.count(); - - const tensors = try allocator.alloc(zml.Tensor, tensor_count); + const tensors = try allocator.alloc(zml.Tensor, store.registry.tensors.count()); defer allocator.free(tensors); - var registry_it = registry.tensors.iterator(); - var load_count: usize = 0; - while (registry_it.next()) |entry| : (load_count += 1) { - tensors[load_count] = switch (mesh_type) { - .replicated => store.view().createReplicatedTensor(entry.key_ptr.*, null), - .sharded => if (entry.value_ptr.shape.rank() > 0) - store.view().createTensor(entry.key_ptr.*, null, .tp, tp_partitioning) + for (tensors, store.registry.tensors.keys(), store.registry.tensors.values()) |*tensor, key, entry| { + tensor.* = switch (mesh_type) { + .replicated => store.view().createReplicatedTensor(key, null), + .sharded => if (entry.shape.rank() > 0) + store.view().createTensor(key, null, .tp, tp_partitioning) else - store.view().createReplicatedTensor(entry.key_ptr.*, null), + store.view().createReplicatedTensor(key, null), }; } const model: AllTensorsModel = .{ .tensors = tensors }; var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); - progress.increaseEstimatedTotalItems(load_count); defer progress.end(); - const now: std.Io.Timestamp = .now(io, .awake); - var buffers = try zml.mem.bufferize(allocator, &model); - defer { - for (buffers.tensors) |*b| b.deinit(); - allocator.free(buffers.tensors); - } - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .parallelism = 8, - .dma_chunks = 16, - .dma_chunk_size = 256 * zml.MiB, - }); - defer loader.deinit(); - - try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + defer zml.mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &store, progress); + try loader.wait(); // logs "Loaded weights" }, } } diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 29c67e0d..b5c8f3ec 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -100,9 +100,6 @@ pub fn main(init: std.process.Init) !void { }, }; - var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); - defer model.deinit(allocator); - // // Load the model and compile it // @@ -112,13 +109,28 @@ pub fn main(init: std.process.Init) !void { var tokenizer = try loadTokenizer(allocator, io, repo, &progress); defer tokenizer.deinit(); + var model: models.LoadedModel = undefined; var compiled_model = try allocator.create(models.CompiledModel); defer allocator.destroy(compiled_model); - compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); - defer compiled_model.deinit(); + var model_buffers = buffers: { + // Returns at once: the loader sets its backend up on a task of its + // own while the model is initialized and compiled. It is freed at + // the end of this block, with its pinned host memory, before the + // chat starts. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = args.model }); + defer loader.deinit(); - // Load buffers after the model compilation to be sure to give enough room to the autotune. - var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); + model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); + errdefer model.deinit(allocator); + + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); + errdefer compiled_model.deinit(); + + // Load buffers after the model compilation to be sure to give enough room to the autotune. + break :buffers try models.LoadedModel.loadBuffers(&model, allocator, &loader, progress, &store); + }; + defer model.deinit(allocator); + defer compiled_model.deinit(); defer zml.mem.deinitBufferized(allocator, &model_buffers); progress.end(); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 851dd856..33e22bf3 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -55,42 +55,24 @@ pub const LoadedModel = union(ModelType) { } } - /// Loads the model's weights from the store. Free them with `zml.mem.deinitBufferized`. + /// Loads the model's weights from the store with `loader`, which logs + /// them. Free them with `zml.mem.deinitBufferized`. pub fn loadBuffers( self: *const LoadedModel, allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, + loader: *zml.io.Loader, + progress: std.Progress.Node, + store: *const zml.io.TensorStore, ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - const buffers: Buffers = switch (self.*) { - inline else => |*m, t| b: { - const Model = @TypeOf(m.inner); + switch (self.*) { + inline else => |*m, t| { var buffers = try zml.mem.bufferize(allocator, &m.inner); errdefer zml.mem.deinitBufferized(allocator, &buffers); - try loader.load(io, Model, &m.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - break :b @unionInit(Buffers, @tagName(t), buffers); + try loader.start(&m.inner, &buffers, store, progress); + try loader.wait(); + return @unionInit(Buffers, @tagName(t), buffers); }, - }; - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; + } } pub fn compile( diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 5ee7aa69..4934d3e6 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -49,12 +49,12 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, .parallelism = 16 }); + var loader: zml.io.Loader = try .init(allocator, io, platform, .{}); defer loader.deinit(); var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); defer zml.mem.deinitBufferized(allocator, &model_buffers); - try loader.load(io, lfm2.Model, &repo_model.inner, &model_buffers, &store, .{ .progress = &progress }); - try loader.await(io); + try loader.start(&repo_model.inner, &model_buffers, &store, progress); + try loader.wait(); const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { .stablehlo => .{ .stablehlo = {} }, diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index c27e6bfd..dae0d054 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -47,12 +47,12 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, .parallelism = 16 }); + var loader: zml.io.Loader = try .init(allocator, io, platform, .{}); defer loader.deinit(); var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); defer zml.mem.deinitBufferized(allocator, &model_buffers); - try loader.load(io, llama.Model, &repo_model.inner, &model_buffers, &store, .{ .progress = &progress }); - try loader.await(io); + try loader.start(&repo_model.inner, &model_buffers, &store, progress); + try loader.wait(); progress.end(); try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index d510e730..878b269b 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -37,32 +37,6 @@ const Mnist = struct { }; } - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, self); - errdefer unloadBuffers(&buffers); - - var loader: zml.io.Loader = try .init(allocator, platform, .default); - errdefer loader.deinit(); - - try loader.load(io, Mnist, self, &buffers, store, .{}); - try loader.await(io); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { - self.fc1.weight.deinit(); - self.fc1.bias.deinit(); - self.fc2.weight.deinit(); - self.fc2.bias.deinit(); - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); @@ -80,24 +54,32 @@ const Mnist = struct { pub fn main(init: std.process.Init) !void { const arena = init.arena; const allocator = init.gpa; - const io = init.io; // Parse program args const process_args = try init.minimal.args.toSlice(arena.allocator()); const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Read model shapes. - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); - defer registry.deinit(); + // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + const io = vfs.io(); // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); - // Init model - var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); + // Returns at once: the loader calibrates while the model is compiled. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = model_path }); + defer loader.deinit(); + + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); defer store.deinit(); + + // Init model const mnist_model: Mnist = .init(store.view()); // // Compile model @@ -110,16 +92,11 @@ pub fn main(init: std.process.Init) !void { }; defer exe.deinit(); - // Load buffers - var mnist_buffers = blk: { - log.info("Transfering weights....", .{}); - const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{f}]", .{ - start.untilNow(io, .awake), - }); - break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); - }; - defer Mnist.unloadBuffers(&mnist_buffers); + // Load weights + var mnist_buffers = try zml.mem.bufferize(allocator, &mnist_model); + defer zml.mem.deinitBufferized(allocator, &mnist_buffers); + try loader.start(&mnist_model, &mnist_buffers, &store, .none); + try loader.wait(); // logs "Loaded weights" var args = try exe.args(allocator); defer args.deinit(allocator); diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index e3871218..76684d94 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -31,25 +31,6 @@ const Mnist = struct { }; } - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, self); - errdefer zml.Buffer.deinitAll(Mnist, &buffers); - - var loader: zml.io.Loader = try .init(allocator, platform, .default); - errdefer loader.deinit(); - - try loader.load(io, Mnist, self, &buffers, store, .{}); - try loader.await(io); - - return buffers; - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); @@ -78,16 +59,16 @@ const Mnist = struct { pub fn main(init: std.process.Init) !void { const arena = init.arena; const allocator = init.gpa; - const io = init.io; // Parse program args const process_args = try init.minimal.args.toSlice(arena.allocator()); const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Read model shapes. - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); - defer registry.deinit(); + // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + const io = vfs.io(); // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); @@ -102,9 +83,16 @@ pub fn main(init: std.process.Init) !void { )); log.info("topology used: {f}", .{dp_mp}); + // Returns at once: the loader calibrates while the model is compiled. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = model_path }); + defer loader.deinit(); + var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); defer profiler.deinit(); + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{dp_mp} }); defer store.deinit(); const mnist_model: Mnist = .init(store.view()); @@ -131,16 +119,11 @@ pub fn main(init: std.process.Init) !void { }; defer exe.deinit(); - // Load buffers - var mnist_buffers = blk: { - log.info("Transfering weights....", .{}); - const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{f}]", .{ - start.untilNow(io, .awake), - }); - break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); - }; - defer zml.Buffer.deinitAll(Mnist, &mnist_buffers); + // Load weights + var mnist_buffers = try zml.mem.bufferize(allocator, &mnist_model); + defer zml.mem.deinitBufferized(allocator, &mnist_buffers); + try loader.start(&mnist_model, &mnist_buffers, &store, .none); + try loader.wait(); // logs "Loaded weights" var args = try exe.args(allocator); defer args.deinit(allocator); From 85b388895ea012908db8dd096d0e0235c679b649 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 19:02:54 +0100 Subject: [PATCH 095/103] docs: show the loader recipe in the README and testing.load in the torch2zml howto The README's taste of ZML called zml.io.load, which does not exist, and a hand-written unloadBuffers. It now ends with a main that creates the loader before compiling, binds the model through a TensorStore view, and loads it with start and wait into buffers freed by deinitBufferized. The torch2zml howto loads its layer with zml.testing.load, the convenience for tests, and frees the weights with deinitBufferized. Co-Authored-By: Claude Opus 5.5 --- README.md | 59 +++++++++++++++++++++------------- docs/howtos/howto_torch2zml.md | 5 +-- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index d9e68b81..34a794a9 100644 --- a/README.md +++ b/README.md @@ -162,29 +162,6 @@ const Mnist = struct { }; } - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - meshes: []const *const zml.Mesh, - ) !zml.Bufferized(Mnist) { - return zml.io.load(Mnist, self, allocator, io, platform, store, .{ - .meshes = meshes, - .parallelism = 1, - .dma_chunks = 1, - .dma_chunk_size = 16 * 1024 * 1024, - }); - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { - self.fc1.weight.deinit(); - self.fc1.bias.deinit(); - self.fc2.weight.deinit(); - self.fc2.bias.deinit(); - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); @@ -195,8 +172,44 @@ const Mnist = struct { return x.argMax(0).indices.convert(.u8); } }; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const path = ...; + + // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + const io = vfs.io(); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + // Returns at once: the loader calibrates while the model is compiled. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); + defer loader.deinit(); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); + defer store.deinit(); + const model: Mnist = .init(store.view()); + + const input: zml.Tensor = .init(.{ 28, 28 }, .u8); + var exe = try platform.compile(allocator, io, model, .forward, .{input}, .{}); + defer exe.deinit(); + + var weights = try zml.mem.bufferize(allocator, &model); + defer zml.mem.deinitBufferized(allocator, &weights); + try loader.start(&model, &weights, &store, .none); + try loader.wait(); // logs "Loaded weights" + + // ... call `exe` on the weights and an image +} ``` +The first `start` waits for whatever calibration is left. + For a full walkthrough, see: - [Getting Started](./docs/tutorials/getting_started.md) diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 69eb19dc..d2c63bc1 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -234,7 +234,7 @@ Python script. Alternatively, Huggingface's web-interface allows to peek into ## Loading an individual layer Currently, the layer is initialized but the weights are still on the disk. -ZML provides a convenience to automatically load weights from an initialized model: +ZML provides a convenience to load weights from an initialized model in a test: ```zig pub fn main(init: std.process.Init) !void { @@ -242,7 +242,8 @@ pub fn main(init: std.process.Init) !void { var platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); - var mlp_weights = try zml.io.load(Mlp, &mlp, allocator, io, platform, &model_store, .auto); + var mlp_weights = try zml.testing.load(allocator, io, platform, &mlp, &model_store); + defer zml.mem.deinitBufferized(allocator, &mlp_weights); } ``` From e7017345b8594b63ce66b6eefed01ee0cef3c648 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 19:03:35 +0100 Subject: [PATCH 096/103] zml/testing: port testLayer to the new store and loader; point core docs at the loader testLayer counted its references by scanning the registry, which the store now owns, and loaded its inputs with zml.io.load, which does not exist, so it compiled only while nothing called it. It now counts the inputs and outputs under their view's prefix, loads the inputs with testing.load, frees them with deinitBufferized, and has a test of its own over a scale layer. Its signature is unchanged. storeOf, load and expectShards name zml.io. Buffer's doc says zml.io.Loader fills buffers from a checkpoint, the extra device memory option names zml.io's hostMappedBytes, and the MoE experts layout doc no longer names a packer. Co-Authored-By: Claude Opus 5.5 --- zml/Compiler.zig | 2 +- zml/buffer.zig | 1 + zml/kernels/moe/moe.zig | 2 +- zml/testing.zig | 80 ++++++++++++++++++++++++----------------- 4 files changed, 51 insertions(+), 34 deletions(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index d3955786..affc2321 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -84,7 +84,7 @@ pub const Options = struct { /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. /// XLA then counts them against device memory, leaves no budget for temporary buffers and /// rematerializes aggressively. Callers set this to the per-device size of such buffers to - /// compensate: `zml.io.next.TensorStore.View.hostMappedBytes` for those the loader delivers + /// compensate: `zml.io.TensorStore.View.hostMappedBytes` for those the loader delivers /// `.host_mapped`. Remove once XLA properly supports host-pinned inputs. extra_device_memory_bytes: u64 = 0, // Debugging options diff --git a/zml/buffer.zig b/zml/buffer.zig index 45bc2be6..ff67ef66 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -27,6 +27,7 @@ test { /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. /// /// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. +/// * filled from a checkpoint by `zml.io.Loader`. /// * created from host data by `fromSlice` or `fromBytes`. pub const Buffer = struct { _platform: *const Platform, diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 25ce26d6..927f95ca 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -24,7 +24,7 @@ pub const ProjectionLayout = enum { concatenated, interleaved }; pub const RoutingWeightPlacement = enum { before_down, after_down }; /// How a backend expects the expert weights to be stored -/// The packer writes this layout and forwardMoe reads it +/// The experts are loaded in this layout and forwardMoe reads it pub const ExpertsLayout = struct { /// Whether the two projections occupy contiguous halves or alternate. gate_up: ProjectionLayout, diff --git a/zml/testing.zig b/zml/testing.zig index 55516494..d1e6faf7 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -475,30 +475,29 @@ pub fn pattern(bytes: []u8, seed: u8) void { for (bytes, 0..) |*byte, index| byte.* = (@as(u8, @truncate(index *% 29)) +% seed *% 53) | 1; } -/// A store of the new loader over a copy of `registry`, for a test that -/// builds several stores over one checkpoint: a store takes its registry's -/// entries. -pub fn storeOf(allocator: std.mem.Allocator, registry: *const zml.safetensors.TensorRegistry, opts: zml.io.next.TensorStore.Options) !zml.io.next.TensorStore { +/// A store over a copy of `registry`, for a test that builds several stores +/// over one checkpoint: a store takes its registry's entries. +pub fn storeOf(allocator: std.mem.Allocator, registry: *const zml.safetensors.TensorRegistry, opts: zml.io.TensorStore.Options) !zml.io.TensorStore { var copy: zml.safetensors.TensorRegistry = .init(allocator); defer copy.deinit(); for (registry.tensors.values()) |tensor| try copy.registerTensor(tensor); return .init(allocator, ©, opts); } -/// Loads `model` from `store` with the new loader, one of its own, and -/// returns its buffers, which the caller frees with `mem.deinitBufferized`. -/// A recipe for tests: production code creates its loader before building -/// the model, so the loader's set-up runs meanwhile. +/// Loads `model` from `store` with a loader of its own and returns its +/// buffers, which the caller frees with `mem.deinitBufferized`. A recipe +/// for tests: production code creates its loader before building the +/// model, so the loader's set-up runs meanwhile. pub fn load( allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, model: anytype, - store: *const zml.io.next.TensorStore, + store: *const zml.io.TensorStore, ) !zml.Bufferized(@TypeOf(model.*)) { var buffers = try zml.mem.bufferize(allocator, model); errdefer zml.mem.deinitBufferized(allocator, &buffers); - var loader = try zml.io.next.Loader.init(allocator, io, platform, .{}); + var loader = try zml.io.Loader.init(allocator, io, platform, .{}); defer loader.deinit(); try loader.start(model, &buffers, store, .none); try loader.wait(); @@ -511,7 +510,7 @@ pub fn load( /// A buffer without a mesh is replicated on the platform's devices. On /// CPU/neuron/metal every memory kind is the one memory space, so only /// CUDA/ROCm/oneAPI tell them apart. -pub fn expectShards(allocator: std.mem.Allocator, io: std.Io, buffer: *const zml.Buffer, global: []const u8, memory: zml.io.next.TensorStore.Memory) !void { +pub fn expectShards(allocator: std.mem.Allocator, io: std.Io, buffer: *const zml.Buffer, global: []const u8, memory: zml.io.TensorStore.Memory) !void { const platform = buffer._platform; const shape = platform.placedShape(buffer.shape()).packedShape(); try std.testing.expectEqual(shape.byteSize(), global.len); @@ -551,18 +550,6 @@ pub fn expectShards(allocator: std.mem.Allocator, io: std.Io, buffer: *const zml } } -fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { - var count: usize = 0; - var it = store.store.registry.iterator(); - while (it.next()) |entry| { - if (std.mem.startsWith(u8, entry.key_ptr.*, prefix)) { - count += 1; - } - } - - return count; -} - /// Test a layer implementation against reference activations stored in a TensorStore. /// The layer is expected to have a method with the name given by `func`. /// The reference activations are expected to be stored under the keys: @@ -601,12 +588,12 @@ pub fn testLayer( const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); const input_count = zml.meta.count(zml.Tensor, &args); - const expected_input_count = countWithPrefix(activation_store, input_name); + const store_input = activation_store.withPrefix(input_name); + const expected_input_count = store_input.count(); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } - const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { @@ -624,7 +611,7 @@ pub fn testLayer( const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); const output_count = exe.output_shapes.len; const store_output = activation_store.withPrefix(output_name); - const expected_output_count = countWithPrefix(store_output, output_name); + const expected_output_count = store_output.count(); if (output_count != expected_output_count) { log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); } @@ -635,12 +622,8 @@ pub fn testLayer( var exe_results = try exe.results(allocator); defer exe_results.deinit(allocator); - var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); - defer zml.meta.visit(struct { - fn cb(_: void, b: *zml.Buffer) void { - b.deinit(); - } - }.cb, {}, &args_buffers); + var args_buffers = try load(allocator, io, platform, &args, activation_store.store); + defer zml.mem.deinitBufferized(allocator, &args_buffers); exe_args.set(.{ layer_weights, args_buffers }); @@ -687,6 +670,39 @@ pub fn testLayer( } } +test testLayer { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = env(); + + const Scale = struct { + weight: zml.Tensor, + + pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { + return x.mul(self.weight); + } + }; + const weight = [_]f32{ 2, -1, 0.5, 3 }; + const input = [_]f32{ 1, 2, 3, 4 }; + const output = [_]f32{ 2, -2, 1.5, 12 }; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: zml.safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try writeSafetensors(allocator, io, tmp.dir, "layer.safetensors", ®istry, &.{ + .{ .name = "scale.weight", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&weight) }, + .{ .name = "x.in.0", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&input) }, + .{ .name = "x.out.0", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&output) }, + }); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); + defer store.deinit(); + + const layer: Scale = .{ .weight = store.view().withPrefix("scale").createReplicatedTensor("weight", null) }; + var layer_weights = try load(allocator, io, platform, &layer, &store); + defer zml.mem.deinitBufferized(allocator, &layer_weights); + try testLayer(allocator, io, platform, layer, .forward, store.view(), "x", layer_weights, &.{}, .{}); +} + fn center(slice: anytype, i: usize) @TypeOf(slice) { const c = 20; const start = if (i < c) 0 else i - c; From 96bb88ba0ac16159eaef8b00d4e51bcb23b52b6f Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 19:06:58 +0100 Subject: [PATCH 097/103] zml/io: delete the old TensorStore, Loader and writers Nothing in zml uses the previous store and loader any more: the borrowing TensorStore with its getReader and maybeCreateBinding, the Loader with load, loadBuffer, loadExecute and await, ProgressWriter, MemoryWriter, BufferedMemoryWriter, DirectShardWriter, DispatchSpans and DirectMemoryWriter, their tests, and the imports only they used. io.zig keeps the VFS and the io/ tree's exports. The mesh validation comment that pointed at DirectMemoryWriter's tests goes with them. Co-Authored-By: Claude Opus 5.5 --- zml/Compiler.zig | 1 - zml/io.zig | 1742 ---------------------------------------------- 2 files changed, 1743 deletions(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index affc2321..d1455789 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -806,7 +806,6 @@ fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { // even if later it means that some mesh axis is never used. // Otherwise a "sharded" tensor can in fact be replicated because the underlying mesh has replicas // TODO(#914): decide if we should forbid this altogether - // currently DirectMemoryWriter tests have replicated meshes depending on the platform they target log.warn("Mesh({s}) has numReplicas() == {d}. This imply that all tensor put on this mesh will be implicitly replicated {d} times.\n{f}", .{ m.name, num_replicas, num_replicas, m }); } log.debug("Using {f}", .{m}); diff --git a/zml/io.zig b/zml/io.zig index e76f47ae..67239d7b 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -20,30 +20,8 @@ const std = @import("std"); -const pjrt = @import("pjrt"); -const stdx = @import("stdx"); pub const VFS = @import("vfs"); -const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("zml.zig").Bufferized; -const CreateOptions = @import("platform.zig").CreateOptions; -const Exe = @import("exe.zig").Exe; -const mem = @import("mem.zig"); -const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); -const pjrtx = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const tracer = @import("profiling/tracer.zig"); -const Quantization = @import("quantization.zig").Quantization; -const safetensors = @import("safetensors.zig"); -const Shape = @import("shape.zig").Shape; -const Sharding = @import("Sharding.zig"); -const Placement = Sharding.Placement; -const Slice = @import("slice.zig").Slice; -const Tensor = @import("tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/io"); - pub const Loader = @import("io/loader.zig").Loader; pub const TensorStore = @import("io/TensorStore.zig"); @@ -56,1723 +34,3 @@ pub const next = @This(); test { std.testing.refAllDecls(@This()); } - -const OldTensorStore = struct { - pub const Binding = struct { - tensors: []*safetensors.Tensor, - transformed: bool, - memory: Memory.Kind = .default, - }; - - registry: *safetensors.TensorRegistry, - meshes: []const *const Sharding.Mesh, - quantization: ?Quantization.Target, - id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - - pub const Options = struct { - /// The meshes `View.createTensor` and `mesh` name. The slice is - /// copied; the meshes must outlive the store. - meshes: []const *const Sharding.Mesh = &.{}, - /// How the platform's FP8 GEMMs want block-FP8 weights, for the code - /// that binds them (`View.quantization`): `.of(platform)`. Required to - /// bind block-FP8 weights. - quantization: ?Quantization.Target = null, - }; - - /// A store over `registry`, which it borrows: the registry must outlive - /// the store, and may still gain entries, which views then find. Each - /// tensor the store binds holds a copy of its entry. - pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, opts: Options) OldTensorStore { - var arena: std.heap.ArenaAllocator = .init(allocator); - return .{ - .registry = registry, - .meshes = arena.allocator().dupe(*const Sharding.Mesh, opts.meshes) catch @panic("OOM"), - .quantization = opts.quantization, - .id_to_sources = .empty, - .allocator = allocator, - .arena = arena, - }; - } - - pub fn deinit(self: *OldTensorStore) void { - self.id_to_sources.deinit(self.allocator); - self.arena.deinit(); - } - - fn putSourcesNoClobber(self: *OldTensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { - const gop = try self.id_to_sources.getOrPut(self.allocator, id); - if (gop.found_existing) { - stdx.debug.panic("Id {} already has associated sources", .{id}); - } - errdefer self.id_to_sources.removeByPtr(gop.key_ptr); - - gop.value_ptr.* = sources; - } - - fn getPtrFromKey(self: *const OldTensorStore, key: []const u8) ?*safetensors.Tensor { - const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; - return tensor_desc_ptr; - } - - fn dupeSource(self: *OldTensorStore, key: []const u8) ?*safetensors.Tensor { - const entry = self.getPtrFromKey(key) orelse return null; - - const copy = self.arena.allocator().create(safetensors.Tensor) catch @panic("OOM"); - copy.* = entry.*; - - return copy; - } - - fn getPtrFromId(self: *const OldTensorStore, id: usize) ?*safetensors.Tensor { - const sources = self.id_to_sources.get(id) orelse return null; - stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); - return sources.tensors[0]; - } - - pub fn getReader(self: *const OldTensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { - return self.registry.reader(io, key, buffer); - } - - pub fn getReaderById(self: *const OldTensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { - const sources = self.id_to_sources.get(id) orelse return error.NotFound; - stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); - - return sources.tensors[0].reader(io, buffer, .{}); - } - - pub fn getSourcesById(self: *const OldTensorStore, id: Tensor.Id) ?Binding { - return self.id_to_sources.get(id); - } - - pub fn view(self: *OldTensorStore) View { - return .{ .store = self }; - } - - pub fn mesh(store: *const OldTensorStore, name: @EnumLiteral()) *const Sharding.Mesh { - const name_slice = @tagName(name); - for (store.meshes) |m| { - if (std.mem.eql(u8, name_slice, m.name)) { - return m; - } - } - std.debug.panic( - \\Found no meshes named {s} in TensorStore. - \\Try passing more meshes to `zml.io.TensorStore.init`. - \\Known meshes: {f} - , .{ name_slice, stdx.fmt.slice(store.meshes) }); - } - - pub const View = struct { - store: *OldTensorStore, - - prefix_buffer: [256]u8 = undefined, - prefix_length: usize = 0, - - pub fn root(self: *const View) View { - return .{ - .store = self.store, - }; - } - - pub fn parent(self: *const View) View { - const slice = self.prefix(); - std.debug.assert(slice.len != 0); - const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); - var buffer: [256]u8 = undefined; - @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = index + 1, - }; - } - - pub fn withPrefix(self: *const View, prefix_: []const u8) View { - var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix(), prefix_ }); - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - pub fn withLayer(self: *const View, index: usize) View { - var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix(), index }); - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - /// The keys' prefix under this view, with its trailing dot; "" at the - /// root. - pub fn prefix(self: *const View) []const u8 { - return self.prefix_buffer[0..self.prefix_length]; - } - - /// The checkpoint tensor `subkey`, the registry's own entry; null - /// when the checkpoint has none. The pointer is valid until the - /// registry next gains an entry. - pub fn get(self: View, subkey: []const u8) ?*const safetensors.Tensor { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); - return self.store.registry.tensors.getPtr(key); - } - - /// See `Options.quantization`. Panics when the store has none: binding - /// block-FP8 weights requires it. - pub fn quantization(self: View) Quantization.Target { - return self.store.quantization orelse - stdx.debug.panic("TensorStore.Options.quantization is unset: block-FP8 weights bind for a platform; pass .of(platform)", .{}); - } - - /// Creates a zml.Tensor from a specific entry in the store. - pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { - return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); - } - - pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { - const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); - const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; - if (mesh_ == .replicated) { - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .replicated); - } - const resolved_mesh = self.store.mesh(mesh_); - // Unspecified tensor axes are treated as replicated - const partition: Sharding.Partitioning = if (has_tags) - Sharding.Partitioning.Logical.init(partitioning).resolve(parsed_tags.constSlice(), resolved_mesh, .replicated) - else - partitioning; - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .{ .mesh = resolved_mesh, .partition = partition }); - } - - pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { - return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); - } - - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { - const tensor = self.maybeCreateTensor(subkey, tags, mesh_, partitioning); - if (tensor) |t| { - const storage = self.store.id_to_sources.getPtr(t.id); - storage.?.memory = .host_pinned; - } - return tensor; - } - - pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { - const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); - const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; - - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.slice() else null, .replicated) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); - } - - fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); - const source = self.store.dupeSource(key) orelse return null; - - const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); - errdefer self.store.arena.allocator().free(sources); - sources[0] = source; - - var shape = source.shape; - if (tags) |user_tags| { - stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); - @memcpy(shape._tags.slice(), user_tags); - } - shape._sharding = sharding; - if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); - - const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); - - return tensor; - } - - pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { - const arena = self.store.arena.allocator(); - - var tensor_list = std.ArrayList(*safetensors.Tensor).initCapacity(arena, sources.len) catch |e| std.debug.panic("Not handling {} errors", .{e}); - defer tensor_list.deinit(arena); - - var buffer: [256]u8 = undefined; - for (sources) |subkey| { - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); - const tensor = self.store.dupeSource(key) orelse return null; - tensor_list.appendAssumeCapacity(tensor); - } - - const tensors = tensor_list.toOwnedSlice(arena) catch unreachable; - errdefer arena.free(tensors); - - const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = tensors, .transformed = true }) catch |e| std.debug.panic("Not handling {} errors", .{e}); - - return tensor; - } - - pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { - var key_buffer: [256]u8 = undefined; - const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix(), subkey }); - return self.store.getReader(key, io, buffer); - } - - pub fn count(self: View) usize { - var count_: usize = 0; - const prefix_ = self.prefix(); - var it = self.store.registry.tensors.iterator(); - while (it.next()) |item| { - const key = item.key_ptr.*; - if (std.mem.startsWith(u8, key, prefix_)) { - count_ += 1; - } - } - return count_; - } - - fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { - const key = std.fmt.bufPrint(buffer, fmt, args) catch - std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); - return key; - } - }; -}; - -const OldLoader = struct { - allocator: std.mem.Allocator, - platform: *const Platform, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - pinned_buffer_pools: []mem.DynamicBufferPool, - group: stdx.Io.LimitedGroup, - bytes_loaded: std.atomic.Value(usize) = .init(0), - delivered: std.AutoHashMapUnmanaged(Tensor.Id, void) = .empty, - - pub const Opts = struct { - pub const default: Opts = .{ - .parallelism = 1, - .dma_chunks = 2, - .dma_chunk_size = 4096, - }; - parallelism: usize, - dma_chunks: usize, - dma_chunk_size: usize, - }; - - pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!OldLoader { - const pool_count = platform.devices.len; - const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); - errdefer allocator.free(dma_allocators); - for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(allocator, device); - } - - const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); - errdefer allocator.free(buffer_pools); - for (buffer_pools) |*pool_| { - pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); - } - errdefer for (buffer_pools, 0..) |*pool_, i| { - pool_.deinit(dma_allocators[i].allocator()); - }; - - return .{ - .allocator = allocator, - .platform = platform, - .dma_allocators = dma_allocators, - .dma_chunk_size = opts.dma_chunk_size, - .pinned_buffer_pools = buffer_pools, - .group = .init(opts.parallelism), - }; - } - - pub fn deinit(self: *OldLoader) void { - self.delivered.deinit(self.allocator); - for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); - self.allocator.free(self.pinned_buffer_pools); - self.allocator.free(self.dma_allocators); - } - - pub fn await(self: *OldLoader, io: std.Io) std.Io.Cancelable!void { - return self.group.await(io); - } - - pub const LoadOpts = struct { - progress: ?*std.Progress.Node = null, - }; - - pub const LoadError = error{TransformedTensorNotDelivered}; - - pub fn load(self: *OldLoader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const OldTensorStore, opts: LoadOpts) LoadError!void { - const tensor_count = meta.count(Tensor, model); - - var arena: std.heap.ArenaAllocator = .init(self.allocator); - defer arena.deinit(); - - const flattened_buffers = arena.allocator().alloc(*Buffer, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); - meta.forEachVisit(buffers, *Buffer, struct { - fn call(i: usize, buffer: *Buffer, flattened_buffers_: []*Buffer) void { - flattened_buffers_[i] = buffer; - } - }.call, .{flattened_buffers}); - - const flattened_tensors = arena.allocator().alloc(*const Tensor, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); - meta.forEachVisit(model, *const Tensor, struct { - fn call(i: usize, tensor: *const Tensor, flattened_tensors_: []*const Tensor) void { - flattened_tensors_[i] = tensor; - } - }.call, .{flattened_tensors}); - - // Refuse before scheduling anything, so a failure leaves no task in flight. - var missing: usize = 0; - for (flattened_tensors) |tensor| { - const sources = store.getSourcesById(tensor.id) orelse continue; - if (!sources.transformed or self.delivered.contains(tensor.id)) { - continue; - } - - missing += 1; - logNotDelivered(arena.allocator(), tensor, sources.tensors); - } - - if (missing != 0) { - return error.TransformedTensorNotDelivered; - } - - for (flattened_tensors, flattened_buffers) |tensor, buffer| { - if (store.getSourcesById(tensor.id)) |sources| { - if (sources.transformed) continue; - } - - self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); - } - } - - fn logNotDelivered(arena: std.mem.Allocator, tensor: *const Tensor, sources: []const *safetensors.Tensor) void { - const max_names = 8; - var names: std.Io.Writer.Allocating = .init(arena); - for (sources[0..@min(sources.len, max_names)], 0..) |source, i| { - names.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break; - } - - if (sources.len > max_names) { - names.writer.print(" and {} more", .{sources.len - max_names}) catch {}; - } - - log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); - } - - pub fn loadBuffer(loader: *OldLoader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const OldTensorStore, opts: LoadOpts) void { - loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); - } - - fn defaultCallback(self: *OldLoader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const OldTensorStore, opts: LoadOpts) void { - const sources = store.getSourcesById(tensor.id) orelse { - std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); - return; - }; - stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); - - self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { - log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); - unreachable; - }; - } - - fn loadSingle(self: *OldLoader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { - self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { - log.err("Failed to load tensor {s}: {}", .{ source.name, e }); - loaded.* = false; - return; - }; - loaded.* = true; - } - - fn loadSingleInner( - self: *OldLoader, - io: std.Io, - source: *safetensors.Tensor, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - opts: LoadOpts, - ) !void { - var reader = try source.reader(io, &.{}, .{}); - defer reader.deinit(); - - var writer = try MemoryWriter.init( - self.allocator, - io, - self.platform, - self.pinned_buffer_pools, - self.dma_allocators, - self.dma_chunk_size, - shape, - buffer, - memory, - ); - defer writer.deinit(self.allocator); - - const scale = 1024; - - if (opts.progress) |progress| { - var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); - defer node.end(); - writer.setProgress(&node); - defer writer.setProgress(null); - var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); - const total = try reader.interface.streamRemaining(&progress_writer.interface); - try progress_writer.interface.flush(); - _ = self.bytes_loaded.fetchAdd(total, .monotonic); - } else { - const total = try reader.interface.streamRemaining(writer.interface()); - try writer.interface().flush(); - _ = self.bytes_loaded.fetchAdd(total, .monotonic); - } - } - - pub fn loadExecute( - self: *OldLoader, - arena: std.mem.Allocator, - io: std.Io, - tensor: Tensor, - buffer: *Buffer, - store: *const OldTensorStore, - exe: *const Exe, - opts: LoadOpts, - ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { - const sources = (store.getSourcesById(tensor.id) orelse return error.NotFound).tensors; - const buffers = try arena.alloc(Buffer, sources.len); - const loaded = try arena.alloc(bool, sources.len); - @memset(loaded, false); - defer for (buffers, loaded) |*b, l| if (l) b.deinit(); - - var node = if (opts.progress) |progress| p: { - var writer = std.Io.Writer.Allocating.init(arena); - print: { - writer.writer.writeAll("Running executable on ") catch break :print; - for (sources, 0..) |source, i| { - writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break :print; - } - } - break :p progress.start(writer.written(), 1); - } else null; - defer if (node) |*n| n.end(); - - for (sources, 0..) |source, i| { - self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); - } - self.group.await(io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - }; - - if (std.mem.findScalar(bool, loaded, false)) |_| { - return error.LoadFailed; - } - - var args = try exe.args(arena); - var results = try exe.results(arena); - - args.set(.{buffers}); - exe.call(io, args, &results, .{ .wait = true }) catch return error.LoadFailed; - - buffer.* = results.get(Buffer); - try self.delivered.put(self.allocator, tensor.id, {}); - } -}; - -test "View.get finds exact keys in the registry the store borrows" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try @import("testing.zig").writeSafetensors(allocator, io, tmp.dir, "layer.safetensors", ®istry, &.{ - .{ .name = "layer.experts.0.down_proj.weight", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, - }); - var store: OldTensorStore = .init(allocator, ®istry, .{ .quantization = .{ .fp8 = .f8e4m3fnuz } }); - defer store.deinit(); - - const layer = store.view().withPrefix("layer"); - const entry = layer.get("experts.0.down_proj.weight").?; - try std.testing.expectEqual(entry, registry.tensors.getPtr("layer.experts.0.down_proj.weight").?); - var bytes: [4]u8 = undefined; - try entry.read(io, &bytes); - try std.testing.expectEqualSlices(u8, &.{ 1, 2, 3, 4 }, &bytes); - // Exact keys only: a key's prefix names no tensor. - try std.testing.expectEqual(null, layer.get("experts.0.down_proj")); - try std.testing.expectEqualStrings("layer.", layer.prefix()); - try std.testing.expectEqualStrings("", layer.root().prefix()); - try std.testing.expectEqual(1, layer.withPrefix("experts").count()); - try std.testing.expectEqual(0, layer.withPrefix("gate").count()); - try std.testing.expect(layer.quantization().fp8.? == .f8e4m3fnuz); - - // An entry registered after `init`, such as an alias of a fused - // tensor, is found too. - try registry.registerTensor(.{ .file_uri = entry.file_uri, .name = "layer.gate.weight", .shape = entry.shape, .offset = entry.offset }); - try std.testing.expect(layer.get("gate.weight") != null); -} - -test "Loader.loadExecute cleans up donated and non-donated inputs" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = @import("testing.zig").env(); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); - const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); - defer allocator.free(path); - - const shape: Shape = .init(.{2}, .i32); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); - try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: OldTensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); - defer store.deinit(); - var loader = try OldLoader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); - defer loader.deinit(); - - inline for (.{ false, true }) |donate| { - const Functions = struct { - fn forward(value: Tensor, weight: Tensor) Tensor { - const sum = value.add(weight); - return if (donate) sum.reuseBuffer(value) else sum; - } - }; - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const tensor = store.view().maybeCreateBinding(&.{ "value", "weight" }, shape) orelse unreachable; - const exe = try platform.compileFn(allocator, io, Functions.forward, .{ Tensor.fromShape(shape), Tensor.fromShape(shape) }, .{}); - defer exe.deinit(); - try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); - - var output: Buffer = undefined; - try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); - defer output.deinit(); - try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); - try std.testing.expect(loader.delivered.contains(tensor.id)); - } -} - -pub const ProgressWriter = struct { - inner: *std.Io.Writer, - progress: *std.Progress.Node, - interface: std.Io.Writer, - total: usize = 0, - scale: usize, - - pub const InitOpts = struct { - scale: usize = 1, - }; - - pub fn init(inner_: *std.Io.Writer, progress_: *std.Progress.Node, opts: InitOpts) ProgressWriter { - return .{ - .inner = inner_, - .progress = progress_, - .scale = opts.scale, - .interface = .{ - .buffer = inner_.buffer, - .end = inner_.end, - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - .sendFile = sendFile, - }, - }, - }; - } - - pub fn pre(self: *ProgressWriter) usize { - self.inner.buffer = self.interface.buffer; - self.inner.end = self.interface.end; - return self.inner.end; - } - - pub fn post(self: *ProgressWriter, len_pre: usize, total: usize) void { - self.interface.buffer = self.inner.buffer; - self.interface.end = self.inner.end; - const drained_pre = len_pre -| self.interface.end; - self.total += drained_pre + total; - self.progress.setCompletedItems(self.total / self.scale); - } - - pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - errdefer self.post(len_pre, 0); - const total = try self.inner.vtable.drain(self.inner, data, splat); - self.post(len_pre, total); - return total; - } - - pub fn sendFile(w: *std.Io.Writer, file_reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.Writer.FileError!usize { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - errdefer self.post(len_pre, 0); - const total = try self.inner.vtable.sendFile(self.inner, file_reader, limit); - self.post(len_pre, total); - return total; - } - - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - defer self.post(len_pre, 0); - try self.inner.vtable.flush(self.inner); - } - - pub fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - defer self.post(len_pre, 0); - try self.inner.vtable.rebase(self.inner, preserve, capacity); - } -}; - -pub const MemoryWriter = union(enum) { - direct: DirectMemoryWriter, - buffered: BufferedMemoryWriter, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - pools: []mem.DynamicBufferPool, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - ) !MemoryWriter { - return switch (platform.target) { - .cuda, .rocm, .oneapi, .furiosa => .{ - .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), - }, - .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) - std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) - else - .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, - }; - } - - pub fn interface(self: *MemoryWriter) *std.Io.Writer { - return switch (self.*) { - .direct => &self.direct.interface, - .buffered => &self.buffered.interface, - }; - } - - pub fn deinit(self: *MemoryWriter, allocator: std.mem.Allocator) void { - switch (self.*) { - .direct => self.direct.deinit(), - .buffered => self.buffered.deinit(allocator), - } - } - - pub fn setProgress(self: *MemoryWriter, progress: ?*std.Progress.Node) void { - switch (self.*) { - .direct => self.direct.setProgress(progress), - .buffered => {}, - } - } -}; - -pub const BufferedMemoryWriter = struct { - io: std.Io, - platform: *const Platform, - shape: Shape, - buffer: *Buffer, - interface: std.Io.Writer, - - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { - return .{ - .io = io, - .platform = platform, - .shape = shape, - .buffer = buffer, - .interface = .{ - .buffer = try allocator.alloc(u8, shape.byteSize()), - .vtable = &.{ - .drain = std.Io.Writer.fixedDrain, - .flush = flush, - .rebase = std.Io.Writer.failingRebase, - }, - }, - }; - } - - pub fn deinit(self: *BufferedMemoryWriter, allocator: std.mem.Allocator) void { - if (self.interface.buffer.len > 0) { - allocator.free(self.interface.buffer); - } - } - - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - self.buffer.* = Buffer.from( - self.io, - self.platform, - self.shape, - @ptrCast(self.interface.buffer), - .{ .wait = true }, - ) catch return std.Io.Writer.Error.WriteFailed; - } -}; - -const DirectShardWriter = struct { - const EventContext = struct { - self: *DirectShardWriter, - err: ?*pjrt.Error = null, - pjrt_event: *pjrt.Event, - event: std.Io.Event = .unset, - buffer: []u8, - }; - - allocator: std.mem.Allocator, - io: std.Io, - memory: *const Memory, - pool: *mem.DynamicBufferPool, - total: usize, - pjrt_buffer: *pjrt.Buffer, - - transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, - offset: usize = 0, - - interface: std.Io.Writer, - flip_flop: u1 = 0, - events_contexts: [2]?EventContext = @splat(null), - - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape) !DirectShardWriter { - const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); - const transfer_manager = try memory.platform.pjrt_client.createBuffersForAsyncHostToDevice( - memory.platform.pjrt_api, - .{ - .shape_specs = &.{shape_spec}, - .memory = memory.pjrt_memory, - }, - ); - errdefer transfer_manager.deinit(memory.platform.pjrt_api); - - const pjrt_buffer = try transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0); - errdefer pjrt_buffer.deinit(memory.platform.pjrt_api); - - const buf = try pool.get(allocator, io); - - return .{ - .allocator = allocator, - .io = io, - .memory = memory, - .pool = pool, - .total = shape.byteSize(), - .pjrt_buffer = pjrt_buffer, - .transfer_manager = transfer_manager, - .interface = .{ - .buffer = buf[0..@min(buf.len, shape.byteSize())], - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - }, - }, - }; - } - - pub fn deinit(self: *DirectShardWriter) void { - self.transfer_manager.deinit(self.memory.platform.pjrt_api); - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - - const chunk = data[0]; - if (chunk.len > self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - if (chunk.len > self.total - (self.offset + self.interface.end)) return std.Io.Writer.Error.WriteFailed; - - const needs_fresh_buffer = chunk.len > self.interface.buffer.len - self.interface.end; - if (needs_fresh_buffer) { - try self.interface.flush(); - } - - @memcpy(self.interface.buffer[self.interface.end..][0..chunk.len], chunk); - self.interface.end += chunk.len; - - const buffer_full = self.interface.end == self.interface.buffer.len; - if (buffer_full) { - try self.interface.flush(); - } - - return chunk.len; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - try self.flushBuffered(); - } - - fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - if (self.interface.buffer.len - self.interface.end >= capacity) return; - if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - try self.interface.flush(); - } - - fn flushBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { - if (self.offset >= self.total) return; - - const pjrt_api = self.memory.platform.pjrt_api; - - const current_buffer = self.interface.buffer; - const buffered = self.interface.buffered(); - - const slice = buffered[0..@min(buffered.len, self.total - self.offset)]; - if (slice.len == 0) return; - const is_last = (self.offset + slice.len) >= self.total; - - const transfer_event = self.transfer_manager.transferData(pjrt_api, 0, slice, @intCast(self.offset), is_last) catch |err| { - log.err("error when transferring data to device: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - - const ctx = &self.events_contexts[@intCast(self.flip_flop)]; - ctx.* = .{ - .self = self, - .buffer = current_buffer, - .pjrt_event = transfer_event, - }; - - transfer_event.onReady(pjrt_api, EventContext, struct { - fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { - ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); - ctx_.err = err; - ctx_.event.set(ctx_.self.io); - } - }.call, &(ctx.*.?)) catch |err| { - log.err("error when setting up transfer completion callback: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - - if (self.events_contexts[@intCast(self.flip_flop ^ 1)]) |*ctx_previous| { - defer self.events_contexts[@intCast(self.flip_flop ^ 1)] = null; - ctx_previous.event.waitUncancelable(self.io); - defer ctx_previous.pjrt_event.deinit(pjrt_api); - if (ctx_previous.err) |e| { - defer e.deinit(pjrt_api); - log.err("error while awaiting: {s}: {s}", .{ - @tagName(e.getCode(pjrt_api)), - e.getMessage(pjrt_api), - }); - return std.Io.Writer.Error.WriteFailed; - } - } - - if (is_last) { - defer ctx.* = null; - defer self.interface = .failing; - const ctx_ = &ctx.*.?; - ctx_.event.waitUncancelable(self.io); - defer ctx_.pjrt_event.deinit(pjrt_api); - if (ctx_.err) |e| { - defer e.deinit(pjrt_api); - log.err("error while awaiting: {s}: {s}", .{ - @tagName(e.getCode(pjrt_api)), - e.getMessage(pjrt_api), - }); - return std.Io.Writer.Error.WriteFailed; - } - } else { - self.interface.end = 0; - const buf = self.pool.get(self.allocator, self.io) catch |err| { - log.err("unable to get a new buffer from the pool: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - self.interface.buffer = buf[0..@min(buf.len, self.total - (self.offset + slice.len))]; - } - self.flip_flop ^= 1; - self.offset += slice.len; - } -}; - -const ShardProgress = struct { - const scale: usize = 1024; - - node: std.Progress.Node, - label: [32]u8 = undefined, - completed: usize = 0, - - fn set(self: *ShardProgress, completed: usize) void { - self.completed = completed; - self.node.setCompletedItems(std.math.divCeil(usize, self.completed, scale) catch unreachable); - } -}; - -// Dispatch planning bridges two different orders: -// -// 1. Placement traversal emits byte ranges per shard writer, in device order. -// That order is convenient for asking "what bytes belong to this shard?", -// but it does not match how a tensor file is read. -// -// 2. Readers stream tensor bytes in global row-major order. DirectMemoryWriter -// must therefore consume dispatch spans in increasing global byte offset. -// -// We first collect placement spans, sort them by global byte range, then fold -// identical ranges into one primary dispatch span plus mirror writers. Identical -// ranges represent replicated shard data: the reader provides those bytes once, -// the primary writer receives them zero-copy, and mirrors receive a copy. -const DispatchSpans = struct { - const DispatchSpan = struct { - start: usize, - end: usize, - primary_writer: usize, - mirror_writer_start: usize, - mirror_writer_len: usize, - }; - - const PlacementSpan = struct { - writer_index: usize, - start: usize, - len: usize, - order: usize, - }; - - spans: []DispatchSpan, - mirror_writers: []usize, - - fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { - const placement = try Sharding.Placement.init(&shape); - const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); - - var placement_span_count: usize = 0; - for (0..ordered_devices.len) |device_index| { - placement_span_count += placementSpanCount(shape, placement.slices(device_index, shape).constSlice()); - } - - var placement_spans: std.ArrayList(PlacementSpan) = try .initCapacity(allocator, placement_span_count); - defer placement_spans.deinit(allocator); - - const byte_strides = shape.computeByteStrides(); - - for (0..ordered_devices.len) |writer_index| { - appendShardPlacementSpans(&placement_spans, shape, placement.slices(writer_index, shape).constSlice(), byte_strides.constSlice(), writer_index); - } - - std.debug.assert(placement_spans.items.len == placement_span_count); - - var spans: std.ArrayList(DispatchSpan) = try .initCapacity(allocator, placement_spans.items.len); - errdefer spans.deinit(allocator); - - var mirror_writers: std.ArrayList(usize) = try .initCapacity(allocator, placement_spans.items.len); - errdefer mirror_writers.deinit(allocator); - - try deduplicateByRange(allocator, placement_spans.items, shape.byteSize(), &spans, &mirror_writers); - - const spans_ = try spans.toOwnedSlice(allocator); - errdefer allocator.free(spans_); - - const mirror_writers_ = try mirror_writers.toOwnedSlice(allocator); - errdefer allocator.free(mirror_writers_); - - return .{ - .spans = spans_, - .mirror_writers = mirror_writers_, - }; - } - - fn deinit(self: DispatchSpans, allocator: std.mem.Allocator) void { - allocator.free(self.spans); - allocator.free(self.mirror_writers); - } - - fn deduplicateByRange( - allocator: std.mem.Allocator, - placement_spans: []PlacementSpan, - total_bytes: usize, - spans: *std.ArrayList(DispatchSpan), - mirror_writers: *std.ArrayList(usize), - ) !void { - const SortContext = struct { - fn lessThan(_: void, lhs: PlacementSpan, rhs: PlacementSpan) bool { - if (lhs.start != rhs.start) return lhs.start < rhs.start; - if (lhs.len != rhs.len) return lhs.len < rhs.len; - return lhs.order < rhs.order; - } - }; - - std.mem.sort(PlacementSpan, placement_spans, {}, SortContext.lessThan); - - var i: usize = 0; - var cursor: usize = 0; - while (i < placement_spans.len) { - const span = placement_spans[i]; - if (span.start != cursor) return error.NonContiguousShardPlacement; - - const mirror_writer_start = mirror_writers.items.len; - var j = i + 1; - while (j < placement_spans.len) : (j += 1) { - const mirror = placement_spans[j]; - if (mirror.start != span.start or mirror.len != span.len) break; - try mirror_writers.append(allocator, mirror.writer_index); - } - - try spans.append(allocator, .{ - .start = span.start, - .end = span.start + span.len, - .primary_writer = span.writer_index, - .mirror_writer_start = mirror_writer_start, - .mirror_writer_len = j - i - 1, - }); - cursor += span.len; - i = j; - } - - if (cursor != total_bytes) return error.NonContiguousShardPlacement; - } - - fn appendPlacementSpan(placement_spans: *std.ArrayList(PlacementSpan), writer_index: usize, start: usize, len: usize) void { - placement_spans.appendAssumeCapacity(.{ - .writer_index = writer_index, - .start = start, - .len = len, - .order = placement_spans.items.len, - }); - } - - fn appendShardPlacementSpans( - placement_spans: *std.ArrayList(PlacementSpan), - shape: Shape, - slices: []const Placement.Slice1d, - byte_strides: []const i64, - writer_index: usize, - ) void { - if (shape.rank() == 0) { - appendPlacementSpan(placement_spans, writer_index, 0, shape.byteSize()); - return; - } - - appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, 0, contiguousSliceAxis(shape, slices), 0); - } - - fn appendShardAxisPlacementSpans( - placement_spans: *std.ArrayList(PlacementSpan), - slices: []const Placement.Slice1d, - byte_strides: []const i64, - writer_index: usize, - axis: usize, - contiguous_axis: usize, - base_start: i64, - ) void { - const slice = slices[axis]; - if (slice.size == 0) return; - - if (axis == contiguous_axis) { - const span_start: usize = @intCast(base_start + slice.start * byte_strides[axis]); - const span_len: usize = @intCast(slice.size * byte_strides[axis]); - appendPlacementSpan(placement_spans, writer_index, span_start, span_len); - return; - } - - var i: i64 = 0; - while (i < slice.size) : (i += 1) { - const child_start = base_start + (slice.start + i) * byte_strides[axis]; - appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, axis + 1, contiguous_axis, child_start); - } - } - - fn placementSpanCount(shape: Shape, slices: []const Placement.Slice1d) usize { - if (shape.rank() == 0) return 1; - - const contiguous_axis = contiguousSliceAxis(shape, slices); - var count: usize = 1; - for (slices[0..contiguous_axis]) |slice| { - count *= @intCast(slice.size); - } - return count; - } - - fn contiguousSliceAxis(shape: Shape, slices: []const Placement.Slice1d) usize { - var axis = shape.rank() - 1; - while (axis > 0) { - const slice = slices[axis]; - if (slice.start != 0 or slice.size != shape.dim(axis)) break; - axis -= 1; - } - return axis; - } -}; - -// Direct load writer state machine. -// -// std.Io.Reader -// | -// v -// DirectMemoryWriter.interface.buffer -// | aliases active primary DirectShardWriter DMA buffer -// v -// commitWindow() -// |-- primary: advance DirectShardWriter.interface.end -// |-- mirrors: copy committed bytes with writeAll() -// |-- boundaries: flush full shard buffers / chunk fence -// v -// DirectShardWriter.flushBuffered() -// | -// v -// PJRT AsyncHostToDeviceTransferManager -// -// The public writer never owns a host staging buffer: it exposes a visible -// prefix of a real shard DMA buffer. The visible prefix may extend beyond the -// current dispatch span to coalesce reader requests, but `commitWindow` must -// scatter those bytes before submitting or rotating the active buffer. The -// first active-primary segment is zero-copy; crossed spans and mirrors treat -// the active buffer as scratch and copy into their own shard writers. -// -// `byte_cursor` is the global tensor position. Shard writer `interface.end` is -// local to the current shard DMA buffer. -pub const DirectMemoryWriter = struct { - allocator: std.mem.Allocator, - shard_writers: []DirectShardWriter, - // Global stream-order spans produced from placement; this is the routing table for committed bytes. - dispatch_spans: DispatchSpans, - // Current entry in `dispatch_spans.spans`; advances whenever `byte_cursor` reaches that span end. - span_index: usize = 0, - // Global tensor byte offset already scattered into shard writers. - byte_cursor: usize = 0, - // Shard writer whose DMA buffer is currently exposed as `interface.buffer`. - active_writer_index: usize, - // Start offset of new reader bytes inside the active shard writer buffer. - window_start: usize, - // Logical maximum public alias window before forcing a cross-shard flush fence. - dma_chunk_size: usize, - shard_progress: ?[]ShardProgress = null, - interface: std.Io.Writer, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - pools: []mem.DynamicBufferPool, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - ) !DirectMemoryWriter { - var placed_shape = shape; - if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; - const placement = try Sharding.Placement.init(&placed_shape); - const mesh = placement.mesh().?; - const ordered_devices = mesh.devicesInCanonicalOrder(); - var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); - errdefer allocator.free(shard_writers); - - var initialized: usize = 0; - errdefer for (shard_writers[0..initialized]) |*writer| { - writer.deinit(); - }; - - var pjrt_buffers: Buffer.Shards = .empty; - for (ordered_devices, 0..) |device, i| { - const pool = &pools[device.id]; - const shard_dma_allocator = dma_allocators[device.id].allocator(); - const pjrt_mem = platform.devices[device.id].memory(memory).?; - - shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.localShape(placed_shape)); - initialized += 1; - - pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); - } - - buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); - - const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); - errdefer dispatch_spans.deinit(allocator); - - const first_span = dispatch_spans.spans[0]; - const first_writer = &shard_writers[first_span.primary_writer]; - const first_window = @min(dma_chunk_size, first_writer.interface.buffer.len); - - return .{ - .allocator = allocator, - .shard_writers = shard_writers, - .dispatch_spans = dispatch_spans, - .active_writer_index = first_span.primary_writer, - .window_start = first_writer.interface.end, - .dma_chunk_size = dma_chunk_size, - .interface = .{ - .buffer = first_writer.interface.buffer[0..first_window], - .end = 0, - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - }, - }, - }; - } - - pub fn deinit(self: *DirectMemoryWriter) void { - self.setProgress(null); - for (self.shard_writers) |*writer| { - writer.deinit(); - } - self.allocator.free(self.shard_writers); - self.dispatch_spans.deinit(self.allocator); - } - - pub fn setProgress(self: *DirectMemoryWriter, progress: ?*std.Progress.Node) void { - const parent = progress orelse { - const states = self.shard_progress orelse return; - for (states) |*s| { - s.node.end(); - } - self.allocator.free(states); - self.shard_progress = null; - return; - }; - - std.debug.assert(self.shard_progress == null); - const states = self.allocator.alloc(ShardProgress, self.shard_writers.len) catch return; - for (states, self.shard_writers, 0..) |*state, writer, i| { - state.completed = 0; - const label = std.fmt.bufPrint(&state.label, "shard[{d}]", .{i}) catch unreachable; - const total_items = std.math.divCeil(usize, writer.total, ShardProgress.scale) catch unreachable; - state.node = parent.start(label, total_items); - state.node.setCompletedItems(0); - } - self.shard_progress = states; - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - const chunk = data[0]; - try self.commitWindow(); - - const writable = self.interface.buffer.len - self.interface.end; - if (writable == 0) return std.Io.Writer.Error.WriteFailed; - - const n = @min(writable, chunk.len); - @memcpy(self.interface.buffer[self.interface.end..][0..n], chunk[0..n]); - self.interface.end += n; - try self.commitWindow(); - - return n; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - // Commit the active public window if one remains - if (self.span_index < self.dispatch_spans.spans.len) { - try self.commitWindow(); - } - - for (self.shard_writers, 0..) |*shard_writer, i| { - try shard_writer.interface.flush(); - if (self.shard_progress) |states| { - states[i].set(shard_writer.total); - } - } - - self.interface = .failing; - } - - fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - if (self.interface.buffer.len - self.interface.end >= capacity) return; - if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - - try self.commitWindow(); - if (self.interface.buffer.len - self.interface.end < capacity) return std.Io.Writer.Error.WriteFailed; - } - - // `interface.buffer` is a clipped alias of the active primary shard DMA - // buffer. Reader writes advance only `DirectMemoryWriter.interface.end`; - // this function scatters that visible range through dispatch spans. - // - // The leading segment for `active_writer_index` is already in the right - // DMA buffer and is committed by advancing that shard writer. Later spans - // use the same bytes as scratch and are copied before any full buffers are - // submitted or the public alias is rotated. - fn commitWindow(self: *DirectMemoryWriter) std.Io.Writer.Error!void { - // All dispatch spans are consumed; no shard DMA buffer remains valid to - // expose as the public alias until flush marks the writer as failing. - if (self.span_index >= self.dispatch_spans.spans.len) { - self.interface.buffer = &.{}; - self.interface.end = 0; - return; - } - - // Only bytes after `window_start` are new reader writes. Bytes before - // it may already be pending in the active shard writer. - const source = self.interface.buffer[self.window_start..self.interface.end]; - if (source.len == 0) return; - - // The visible alias window may cross many dispatch spans. Walk the - // global cursor and route each subrange to its primary and mirrors. - var consumed: usize = 0; - while (consumed < source.len) { - if (self.span_index >= self.dispatch_spans.spans.len) return std.Io.Writer.Error.WriteFailed; - - const span = self.dispatch_spans.spans[self.span_index]; - if (self.byte_cursor < span.start or self.byte_cursor >= span.end) return std.Io.Writer.Error.WriteFailed; - - const n = @min(source.len - consumed, span.end - self.byte_cursor); - const chunk = source[consumed..][0..n]; - - // The first span in the active alias is already in the primary DMA - // buffer. Later ranges use the active buffer as scratch and must be - // copied into their real destination writer. - if (span.primary_writer == self.active_writer_index and consumed == 0) { - self.shard_writers[span.primary_writer].interface.end += n; - } else { - const primary_writer = &self.shard_writers[span.primary_writer]; - if (span.primary_writer == self.active_writer_index) { - @memmove(primary_writer.interface.buffer[primary_writer.interface.end..][0..n], chunk); - primary_writer.interface.end += n; - } else { - try primary_writer.interface.writeAll(chunk); - } - } - if (self.shard_progress) |states| { - states[span.primary_writer].set(states[span.primary_writer].completed + n); - } - - const mirror_writer_end = span.mirror_writer_start + span.mirror_writer_len; - for (self.dispatch_spans.mirror_writers[span.mirror_writer_start..mirror_writer_end]) |mirror_writer_index| { - const mirror_writer = &self.shard_writers[mirror_writer_index]; - if (mirror_writer_index == self.active_writer_index) { - @memmove(mirror_writer.interface.buffer[mirror_writer.interface.end..][0..n], chunk); - mirror_writer.interface.end += n; - } else { - try mirror_writer.interface.writeAll(chunk); - } - if (self.shard_progress) |states| { - states[mirror_writer_index].set(states[mirror_writer_index].completed + n); - } - } - - self.byte_cursor += n; - consumed += n; - if (self.byte_cursor == span.end) { - self.span_index += 1; - } - } - - // All scratch copies are complete now, so full DMA buffers may be - // submitted without invalidating bytes still needed for scatter. - for (self.shard_writers) |*writer| { - if (writer.interface.end == writer.interface.buffer.len) { - try writer.interface.flush(); - } - } - - if (@mod(self.byte_cursor, self.dma_chunk_size) == 0) { - for (self.shard_writers) |*writer| { - try writer.interface.flush(); - } - } - - // The last committed window may finish the tensor exactly. Leave an - // empty public window so any further writes fail through std.Io. - if (self.span_index >= self.dispatch_spans.spans.len) { - self.interface.buffer = &.{}; - self.interface.end = 0; - return; - } - - // Rotate the public alias to the primary writer for the next stream - // span. `window_start` preserves that writer's already-buffered prefix - // so only newly advanced bytes are scattered on the next commit. - const next_span = self.dispatch_spans.spans[self.span_index]; - const next_writer = &self.shard_writers[next_span.primary_writer]; - self.active_writer_index = next_span.primary_writer; - self.window_start = next_writer.interface.end; - - // Publish the next public alias. Its backing memory is the next primary - // shard buffer, but the visible length is allowed to coalesce across - // upcoming spans until a buffer or tensor boundary. - const total = self.dispatch_spans.spans[self.dispatch_spans.spans.len - 1].end; - const buffer_remaining = next_writer.interface.buffer.len - self.window_start; - const chunk_offset = @mod(self.byte_cursor, self.dma_chunk_size); - const chunk_remaining = if (chunk_offset == 0) self.dma_chunk_size else self.dma_chunk_size - chunk_offset; - const tensor_remaining = total - self.byte_cursor; - const visible_remaining = @min(buffer_remaining, @min(chunk_remaining, tensor_remaining)); - if (visible_remaining == 0) return std.Io.Writer.Error.WriteFailed; - - self.interface.buffer = next_writer.interface.buffer[0 .. self.window_start + visible_remaining]; - self.interface.end = self.window_start; - } -}; - -const DirectMemoryWriterDeviceTest = struct { - const WriteMode = enum { - stream_remaining, - writable_slice_greedy, - }; - - pub const Scenario = struct { - name: []const u8, - create_options: CreateOptions, - shape: Shape, - logical_mesh: Sharding.LogicalMesh, - strategy: Sharding.Strategy, - write_mode: WriteMode = .stream_remaining, - writable_slice_min_len: usize = 128, - pool_chunks: usize = 4, - pool_chunk_size: usize = 1 << 20, - memory: Memory.Kind = .default, - }; - - allocator: std.mem.Allocator, - io: std.Io, - - fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { - var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; - defer platform.deinit(self.allocator, self.io); - - const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); - std.log.warn("mesh: {f}", .{mesh}); - try self.runDirectMemoryWriter( - platform, - scenario.shape.withPartitioning(&mesh, partitioning), - scenario.write_mode, - scenario.writable_slice_min_len, - scenario.pool_chunks, - scenario.pool_chunk_size, - scenario.memory, - ); - } - - fn runDirectMemoryWriter( - self: DirectMemoryWriterDeviceTest, - platform: *const Platform, - shape: Shape, - write_mode: WriteMode, - writable_slice_min_len: usize, - pool_chunks: usize, - pool_chunk_size: usize, - memory: Memory.Kind, - ) !void { - const slice = try Slice.alloc(self.allocator, shape); - defer slice.free(self.allocator); - - for (slice.items(f32), 0..) |*e, i| { - e.* = @as(f32, @floatFromInt(i)); - } - - const pool_count = platform.devices.len; - const dma_allocators = try self.allocator.alloc(mem.DmaAllocator, pool_count); - defer self.allocator.free(dma_allocators); - for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(self.allocator, device); - } - - const pools = try self.allocator.alloc(mem.DynamicBufferPool, pool_count); - defer self.allocator.free(pools); - for (pools) |*pool| { - pool.* = .init(pool_chunks, pool_chunk_size); - } - defer for (pools, 0..) |*pool, i| { - pool.deinit(dma_allocators[i].allocator()); - }; - - var written_buffer: Buffer = undefined; - var writer: DirectMemoryWriter = try .init( - self.allocator, - self.io, - platform, - pools, - dma_allocators, - pool_chunk_size, - shape, - &written_buffer, - memory, - ); - defer writer.deinit(); - defer written_buffer.deinit(); - - switch (write_mode) { - .stream_remaining => { - var reader: std.Io.Reader = .fixed(slice.constData()); - const streamed = try reader.streamRemaining(&writer.interface); - try std.testing.expectEqual(slice.constData().len, streamed); - }, - .writable_slice_greedy => { - var offset: usize = 0; - while (offset < slice.constData().len) { - const min_len = @max(@as(usize, 1), writable_slice_min_len); - const dest = try writer.interface.writableSliceGreedy(min_len); - const to_write = @min(dest.len, slice.constData().len - offset); - if (to_write == 0) return std.Io.Writer.Error.WriteFailed; - @memcpy(dest[0..to_write], slice.constData()[offset..][0..to_write]); - writer.interface.advance(to_write); - offset += to_write; - } - }, - } - - try writer.interface.flush(); - try written_buffer.await(self.io); - - var written_slice = try written_buffer.toSliceAlloc(self.allocator, self.io); - defer written_slice.free(self.allocator); - try std.testing.expectEqualSlices(u8, slice.constData(), written_slice.constData()); - } -}; - -test "DirectMemoryWriter: replicated tensor" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "replicated_auto", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), - .logical_mesh = .mesh(.{ .x = .high_bandwidth }), - .strategy = .parseBindings(.{ .x = .link_x }), - }, .{ .rows = .replicated, .cols = .replicated }); -} - -test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_auto", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - }, .{ .rows = .replicated, .cols = .model }); -} - -test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "batch_model_2d_torus", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), - .logical_mesh = .mesh(.{ - .batch = .low_bandwidth, - .model = .high_bandwidth, - }), - .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), - }, .{ .batch = .batch, .model = .model }); -} - -test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_folded_2d_torus", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_y }); - break :blk strategy; - }, - }, .{ .model = .model }); -} - -test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_auto_writable_slice", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - .write_mode = .writable_slice_greedy, - .writable_slice_min_len = 64, - .pool_chunk_size = 1024, - }, .{ .rows = .replicated, .cols = .model }); -} - -test "DirectMemoryWriter: 3D topology folded model + replicated batch" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "topology_3d_folded_model", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, - .cpu = .{ .device_count = 8 }, - }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ - .batch = .low_bandwidth, - .model = .high_bandwidth, - }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_z }); - break :blk strategy; - }, - }, .{ .batch = .replicated, .model = .model }); -} - -test "MemoryWriter can produce a host pinned buffer" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "host_pinned", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus1d }, - }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - .memory = .host_pinned, - }, .{ .batch = .replicated, .model = .model }); -} From d6376486bf7db84e65c28fa5b8793b071592a212 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 19:09:28 +0100 Subject: [PATCH 098/103] zml: delete TensorReader, the DMA allocators, LimitedGroup and the unloadBuffers helpers These served only the deleted loader or its callers: - safetensors.TensorReader, Tensor.reader and the registry's reader, iterator and totalBytes; Tensor.read reads a whole tensor, and a store's views look tensors up; - mem's DmaAllocator, UninitializedBufferAllocator, DmaMapAllocator and DynamicBufferPool: the direct backend pins its own host memory, so mem.zig keeps only the buffer-tree helpers; - stdx.Io.LimitedGroup; - nn.Linear.unloadBuffers and Buffer.freeDeviceMemoryButKeepHostMetadataMemory: mem.deinitBufferized frees a whole tree, without a hand-written walker that misses a field. Co-Authored-By: Claude Opus 5.5 --- stdx/Io.zig | 58 ---------- zml/buffer.zig | 11 -- zml/mem.zig | 267 -------------------------------------------- zml/nn.zig | 4 - zml/safetensors.zig | 193 -------------------------------- 5 files changed, 533 deletions(-) diff --git a/stdx/Io.zig b/stdx/Io.zig index f1ae8669..453ee503 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -22,64 +22,6 @@ pub const Dir = struct { } }; -pub const LimitedGroup = struct { - limit: usize, - in_flight: std.atomic.Value(usize) = .init(0), - group: std.Io.Group = .init, - cond: std.Io.Condition = .init, - mutex: std.Io.Mutex = .init, - - fn Wrapper(comptime function: anytype) type { - return struct { - fn wrapper(self: *LimitedGroup, io: std.Io, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.Cancelable!void { - while (true) { - var in_flight = self.in_flight.load(.acquire); - while (in_flight < self.limit) { - if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { - in_flight = actual; - continue; - } - - defer { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - _ = self.in_flight.fetchSub(1, .release); - self.cond.signal(io); - } - return @call(.auto, function, args); - } - - try self.mutex.lock(io); - defer self.mutex.unlock(io); - while (self.in_flight.load(.acquire) >= self.limit) { - try self.cond.wait(io, &self.mutex); - } - } - } - }; - } - - pub fn init(limit: usize) LimitedGroup { - return .{ .limit = limit }; - } - - pub fn async(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void { - self.group.async(io, Wrapper(function).wrapper, .{ self, io, args }); - } - - pub fn concurrent(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.ConcurrentError!void { - try self.group.concurrent(io, Wrapper(function).wrapper, .{ self, io, args }); - } - - pub fn await(self: *LimitedGroup, io: std.Io) std.Io.Cancelable!void { - try self.group.await(io); - } - - pub fn cancel(self: *LimitedGroup, io: std.Io) void { - self.group.cancel(io); - } -}; - pub const Duration = struct { pub fn div(self: std.Io.Duration, rhs: u64) std.Io.Duration { return .fromNanoseconds(@divTrunc(self.nanoseconds, @as(i96, @intCast(rhs)))); diff --git a/zml/buffer.zig b/zml/buffer.zig index ff67ef66..288d0c90 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -109,17 +109,6 @@ pub const Buffer = struct { }.deinit, {}, buffers); } - /// Given an arbitrary struct `deinit` all `zml.Buffer` containing. - /// If the struct contains slices of `zml.Buffer` the memory of the slices will NOT be freed, - /// This only impacts device memory. - pub fn freeDeviceMemoryButKeepHostMetadataMemory(T: type, buffers: *mem.Bufferized(T)) void { - meta.visit(struct { - fn deinit(_: void, x: *Buffer) void { - x.deinit(); - } - }.deinit, {}, buffers); - } - /// This Buffer shape. pub fn shape(self: Buffer) Shape { return self._shape; diff --git a/zml/mem.zig b/zml/mem.zig index dab7a87d..596ebe85 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -1,276 +1,9 @@ const std = @import("std"); -const Allocator = std.mem.Allocator; -const Alignment = std.mem.Alignment; -const assert = std.debug.assert; - -const pjrt = @import("pjrt"); const Buffer = @import("buffer.zig").Buffer; -const Device = @import("platform.zig").Device; -const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); -const Platform = @import("platform.zig").Platform; const Tensor = @import("tensor.zig").Tensor; -const log = std.log.scoped(.@"zml/mem"); - -pub const DmaAllocator = union(enum) { - passthrough: std.mem.Allocator, - uib: UninitializedBufferAllocator, - dmam: DmaMapAllocator, - - pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { - return switch (device.platform.target) { - .cuda => .{ .dmam = .init(parent, device.platform) }, - .rocm, .oneapi, .tpu, .furiosa => .{ .uib = .init(device.memory(.host_pinned).?) }, - .cpu, .neuron, .metal => .{ .passthrough = parent }, - }; - } - - pub fn allocator(self: *const DmaAllocator) std.mem.Allocator { - return switch (self.*) { - .passthrough => |a| a, - inline else => |*a| a.allocator(), - }; - } -}; - -pub const UninitializedBufferAllocator = struct { - memory: *const Memory, - - const Header = struct { - buffer: *pjrt.Buffer, - }; - - pub fn init(memory: *const Memory) UninitializedBufferAllocator { - return .{ - .memory = memory, - }; - } - - pub fn allocator(self: *const UninitializedBufferAllocator) std.mem.Allocator { - return .{ - .ptr = @constCast(self), - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }, - }; - } - - fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { - const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); - const pjrt_api = self.memory.platform.pjrt_api; - const pjrt_client = self.memory.platform.pjrt_client; - - const total_len = std.mem.alignForward(usize, @sizeOf(Header) + len, alignment.toByteUnits()); - - const pjrt_buffer = pjrt_client.createUninitializedBuffer(pjrt_api, .{ - .dims = &.{@intCast(total_len)}, - .element_type = .u8, - .layout = .{ - .tiled = .{ - .minor_to_major = &.{0}, - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - .dst = .{ .memory = self.memory.pjrt_memory }, - }) catch return null; - - const opaque_ptr: [*]u8 = @ptrCast(pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable); - const data_with_header: []u8 = opaque_ptr[0..total_len]; - - const header = std.mem.bytesAsValue(Header, opaque_ptr); - header.* = .{ - .buffer = pjrt_buffer, - }; - const offset = std.mem.alignForward(usize, @sizeOf(Header), alignment.toByteUnits()); - return @ptrCast(data_with_header[offset..]); - } - - fn free(ctx: *anyopaque, buf: []u8, alignment: Alignment, ret_addr: usize) void { - _ = ret_addr; - const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); - const pjrt_api = self.memory.platform.pjrt_api; - const header: *Header = @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(buf.ptr) - @sizeOf(Header), alignment.toByteUnits())); - header.buffer.deinit(pjrt_api); - } - - fn resize(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return false; - } - - fn remap(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return null; - } -}; - -pub const DmaMapAllocator = struct { - parent: std.mem.Allocator, - platform: *const Platform, - - pub fn init(parent: std.mem.Allocator, platform: *const Platform) DmaMapAllocator { - return .{ - .parent = parent, - .platform = platform, - }; - } - - pub fn allocator(self: *const DmaMapAllocator) Allocator { - return .{ - .ptr = @constCast(self), - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }, - }; - } - - fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { - const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); - const allocation = self.parent.rawAlloc(len, alignment, ret_addr); - if (allocation) |loc| { - const data = loc[0..len]; - self.platform.pjrt_client.dmaMap(self.platform.pjrt_api, @ptrCast(data)) catch { - self.parent.rawFree(data, alignment, ret_addr); - return null; - }; - } - return allocation; - } - - fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return false; - } - - fn remap(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return null; - } - - fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { - const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); - self.platform.pjrt_client.dmaUnmap(self.platform.pjrt_api, @ptrCast(buf[0..buf.len])) catch unreachable; - self.parent.rawFree(buf, alignment, ret_addr); - } -}; - -pub const DynamicBufferPool = struct { - const Node = struct { next: ?*Node }; - const alignment: std.mem.Alignment = .of(Node); - - block_size: usize, - max_blocks: usize, - - arena_state: std.heap.ArenaAllocator.State = .{}, - free_stack: std.atomic.Value(?*Node) = .init(null), - in_flight: std.atomic.Value(usize) = .init(0), - mutex: std.Io.Mutex = .init, - condition: std.Io.Condition = .init, - - pub fn init(max_blocks: usize, block_size: usize) DynamicBufferPool { - return .{ - .block_size = block_size, - .max_blocks = max_blocks, - }; - } - - pub fn deinit(self: *DynamicBufferPool, allocator: std.mem.Allocator) void { - self.arena_state.promote(allocator).deinit(); - } - - pub fn get(self: *DynamicBufferPool, allocator: std.mem.Allocator, io: std.Io) ![]u8 { - while (true) { - var in_flight = self.in_flight.load(.acquire); - while (in_flight < self.max_blocks) { - if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { - in_flight = actual; - continue; - } - - if (self.pop()) |node| { - return self.nodeToSlice(node); - } - - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - var arena = self.arena_state.promote(allocator); - defer self.arena_state = arena.state; - - errdefer _ = self.in_flight.fetchSub(1, .release); - const buffer = try arena.allocator().alignedAlloc(u8, alignment, self.block_size); - return @alignCast(buffer); - } - - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - while (self.in_flight.load(.acquire) >= self.max_blocks) { - try self.condition.wait(io, &self.mutex); - } - } - } - - pub fn put(self: *DynamicBufferPool, io: std.Io, buf: []u8) void { - const node: *Node = @ptrCast(@alignCast(buf.ptr)); - var head = self.free_stack.load(.acquire); - while (true) { - node.next = head; - if (self.free_stack.cmpxchgWeak(head, node, .release, .acquire)) |actual_head| { - head = actual_head; - } else { - break; - } - } - - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - _ = self.in_flight.fetchSub(1, .release); - self.condition.signal(io); - } - - fn pop(self: *DynamicBufferPool) ?*Node { - var head = self.free_stack.load(.acquire); - while (true) { - if (self.free_stack.cmpxchgWeak(head, if (head) |h| h.next else null, .acquire, .acquire)) |actual| { - head = actual; - continue; - } - - return head; - } - } - - fn nodeToSlice(self: DynamicBufferPool, node: *Node) []u8 { - const ptr: [*]u8 = @ptrCast(node); - return ptr[0..self.block_size]; - } -}; - /// Return a clone of a type with Tensors replaced by Buffer. /// Non-Tensor metadata is stripped out of the resulting struct. /// Recursively descends into the type. diff --git a/zml/nn.zig b/zml/nn.zig index 00501d55..e8d9873d 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -31,10 +31,6 @@ pub const Linear = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { - zml.Buffer.deinitAll(Linear, self); - } - pub fn quantizationScheme(self: Linear) ?Quantization.Scheme { return if (self.quantization) |q| q.scheme else null; } diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 5a10aa1b..beee2cca 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -10,166 +10,6 @@ const log = std.log.scoped(.@"zml/safetensors"); const BYTES_HEADER = 8; -pub const TensorReader = struct { - tensor: Tensor, - file: std.Io.File, - file_reader: std.Io.File.Reader, - alignment: ?std.mem.Alignment, - remaining: u64, - padding_remaining: u64, - - io: std.Io, - interface: std.Io.Reader, - - pub const InitOpts = struct { - alignment: ?std.mem.Alignment = null, - }; - - pub fn init( - io: std.Io, - tensor: Tensor, - buffer: []u8, - opts: InitOpts, - ) !TensorReader { - const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); - errdefer file.close(io); - - const offset, const padding_remaining = if (opts.alignment) |alignment| blk: { - const aligned_offset = std.mem.alignBackward(u64, tensor.offset, alignment.toByteUnits()); - break :blk .{ aligned_offset, tensor.offset - aligned_offset }; - } else blk: { - break :blk .{ tensor.offset, 0 }; - }; - - var file_reader = file.reader(io, buffer); - try file_reader.seekTo(offset); - - return .{ - .tensor = tensor, - .file = file, - .file_reader = file_reader, - .alignment = opts.alignment, - .remaining = tensor.byteSize(), - .padding_remaining = padding_remaining, - .io = io, - .interface = .{ - .vtable = if (opts.alignment) |_| blk: { - break :blk &.{ - .stream = streamAligned, - .discard = discardAligned, - }; - } else blk: { - break :blk &.{ - .stream = stream, - .discard = discard, - }; - }, - .buffer = &.{}, - .seek = 0, - .end = 0, - }, - }; - } - - pub fn deinit(self: *TensorReader) void { - self.file.close(self.io); - } - - // Limit single reads to 1GB to avoid issues on macOS where pread returns - // INT_MAX when the requested size is larger than 2GB - const max_single_read_bytes: u64 = 1 << 30; - - fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const combined_limit = limit.min(.limited64(self.remaining)).min(.limited64(max_single_read_bytes)); - const n = try self.file_reader.interface.stream(w, combined_limit); - self.remaining -= n; - return n; - } - - fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const combined_limit = limit.min(.limited64(self.remaining)); - const n = try self.file_reader.interface.discard(combined_limit); - self.remaining -= n; - return n; - } - - fn streamAligned(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const requested_bytes = @min(limit.minInt64(self.remaining), max_single_read_bytes); - - if (self.padding_remaining > 0) { - _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); - self.padding_remaining = 0; - } - - const align_bytes = self.alignment.?.toByteUnits(); - const file_offset = self.file_reader.pos - std.mem.alignBackward(u64, self.file_reader.pos, align_bytes); - const writer_addr = @intFromPtr(w.buffer.ptr) + w.end; - const writer_offset = writer_addr - std.mem.alignBackward(usize, writer_addr, align_bytes); - - if (file_offset != writer_offset) { - const skip = if (file_offset >= writer_offset) blk: { - break :blk file_offset - writer_offset; - } else blk: { - break :blk align_bytes - writer_offset + file_offset; - }; - - w.advance(skip); - } - - if (w.writableSliceGreedy(align_bytes)) |dest| { - const phys_buf = dest.ptr - file_offset; - const phys_pos = self.file_reader.pos - file_offset; - const phys_size = std.mem.alignBackward(usize, @min(dest.len, requested_bytes) + file_offset, align_bytes); - - if (phys_size > file_offset) { - const phys_read = self.file.readPositional(self.io, &.{phys_buf[0..phys_size]}, phys_pos) catch unreachable; - if (phys_read > file_offset) { - const bytes_read = phys_read - file_offset; - w.advance(bytes_read); - self.file_reader.pos += bytes_read; - self.remaining -= bytes_read; - return bytes_read; - } - } - } else |_| {} - - const bytes_read = try self.file_reader.interface.stream(w, .limited64(requested_bytes)); - self.remaining -= bytes_read; - - return bytes_read; - } - - fn discardAligned(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) return 0; - - if (self.padding_remaining > 0) { - _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); - self.padding_remaining = 0; - } - - const combined_limit = limit.min(.limited64(self.remaining)); - const n = try self.file_reader.interface.discard(combined_limit); - self.remaining -= n; - return n; - } -}; - /// A checkpoint tensor: its name, its shape, and where its bytes are, at /// `offset` of the file `file_uri` names. The URI is what the io that read /// the header resolved, a VFS URI under a VFS, so read the tensor through @@ -185,10 +25,6 @@ pub const Tensor = struct { return self.shape.byteSize(); } - pub fn reader(self: Tensor, io: std.Io, buffer: []u8, opts: TensorReader.InitOpts) !TensorReader { - return TensorReader.init(io, self, buffer, opts); - } - /// Reads the whole tensor into `buffer`, which holds exactly its bytes /// (`OutOfBounds` otherwise). Opens `file_uri` through `io` for the /// call, failing as that io's open does; a file that ends inside the @@ -403,35 +239,6 @@ pub const TensorRegistry = struct { } entry.value_ptr.* = tensor_copy; } - - pub fn reader( - self: *TensorRegistry, - io: std.Io, - tensor_name: []const u8, - buffer: []u8, - ) !TensorReader { - const tensor = self.tensors.get(tensor_name) orelse { - log.err("Tensor {s} not found in registry", .{tensor_name}); - return error.TensorNotFound; - }; - - return try .init(io, tensor, buffer, .{}); - } - - pub fn iterator(self: *TensorRegistry) Tensors.Iterator { - return self.tensors.iterator(); - } - - pub fn totalBytes(self: *TensorRegistry) u64 { - var total: u64 = 0; - - var it = self.tensors.iterator(); - while (it.next()) |entry| { - total += entry.value_ptr.byteSize(); - } - - return total; - } }; /// What a checkpoint's entrypoint is: an index of the safetensors files From 9e8659fe7f2e056e35a3b1504c2bec5d0b54786d Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 4 Nov 2026 01:34:25 +0100 Subject: [PATCH 099/103] vfs/s3: move S3 onto the governed request loop S3's lookups, listing and reads now run through request.zig. S3.Options holds the keys, endpoint and region of the old Config plus a retry policy, and replaces Config and InitOpts; requests are always signed for the s3 service. The governor, named s3, treats a 503 as a throttle, since S3 answers SlowDown with it, so it holds the backend's requests instead of charging retries. Each attempt signs anew, because the SigV4 signature covers x-amz-date and the Range header: the HEAD that sizes an object, the bucket listing, and the one Range GET per positional read. The 32-worker pool that split each read into 16 MiB GETs goes: the reader's width is the backend's concurrency. Opening, stat-ing and probing a key follow the open contract, so a 403 is PermissionDenied, which is also what S3 answers a principal without s3:ListBucket for a missing key. access sends a HEAD where it succeeded for any path, a canceled listing or read stays Canceled, and any other failed listing is logged once. A seek back saturates at 0 instead of overflowing on minInt(i64), and deinit logs the read totals at debug. The acceptance tests run the open contract, the refused streaming read and seek, and the 503 hold on S3 beside HTTP, and the VFS's test of refused writes registers S3 on the mock server and checks that no request reaches it and nothing lands on local disk, where S3's base layer would write. A unit test pins the SigV4 Authorization of the HEAD, the listing and a Range GET, at a fixed date, to botocore's. Checked against a local MinIO: signed HEAD, listing and Range GETs succeed, a wrong secret or a bucket the principal may not read opens as PermissionDenied (Unexpected before), and a missing key as FileNotFound. load_check of LiquidAI/LFM2.5-1.2B-Thinking over s3:// on CPU, 8 interleaved runs against the pooled backend: 2.18 GiB in 310.6 ms median (302.7-316.4) against 306.9 (294.4-324.8), with the same 3 HEADs and 141 GETs and no retries; bytes match on every run. With the byte check, whose reads are one tensor at a time, the pool's split made the check faster: 1.58 s against 1.32 s median, 289 GETs against 362. MinIO answers 404 for a missing key whatever the principal's s3:ListBucket, so the 403 for a missing key was checked on the mock only; real AWS was not reachable from here. Co-Authored-By: Claude Opus 5.5 --- vfs/http_acceptance_test.zig | 152 +++++++--- vfs/s3.zig | 534 +++++++++++++---------------------- vfs/vfs.zig | 57 ++-- 3 files changed, 334 insertions(+), 409 deletions(-) diff --git a/vfs/http_acceptance_test.zig b/vfs/http_acceptance_test.zig index d79bfcbb..eca825fc 100644 --- a/vfs/http_acceptance_test.zig +++ b/vfs/http_acceptance_test.zig @@ -4,6 +4,7 @@ const HTTP = @import("http.zig").HTTP; const MockServer = @import("mock_server.zig").MockServer; const range_read = @import("range_read.zig"); const vfs_request = @import("request.zig"); +const S3 = @import("s3.zig").S3; /// A retry configuration that never sleeps, for the tests that count /// attempts rather than time. @@ -397,7 +398,7 @@ test "a throttled HEAD at open waits the hold out and succeeds" { try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); } -test "a 503 retries a plain HTTP server without holding it" { +test "a 503 holds an object store and only retries a plain HTTP server" { const allocator = std.testing.allocator; const io = std.testing.io; const read_size = 512; @@ -406,30 +407,53 @@ test "a 503 retries a plain HTTP server without holding it" { @memset(&object, 0x7e); var output: [read_size]u8 = undefined; - // The first two GETs are answered 503. + // Each backend's first two GETs are answered 503. var server = try MockServer.init(io, &object, .{ .throttle = .{ .first_gets = 2, .status = .service_unavailable }, + .objects = &.{ + .{ .path = "/bucket/object", .bytes = &object }, + }, }); defer server.deinit(io); try server.start(io); var client: std.http.Client = .{ .allocator = allocator, .io = io }; defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(5) }); + defer s3.deinit(); var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(5) }); defer http.deinit(); - const http_io = http.io(); var path_buffer: [128]u8 = undefined; - const file = try std.Io.Dir.openFile(.cwd(), http_io, try server.path(&path_buffer), .{}); - defer file.close(http_io); - try std.testing.expectEqual(read_size, try file.readPositional(http_io, &.{&output}, 0)); + const backends = [_]struct { std.Io, *const vfs_request.Governor, []const u8, bool }{ + .{ s3.io(), &s3.governor, "bucket/object", true }, + .{ http.io(), &http.governor, try server.path(&path_buffer), false }, + }; + for (backends) |backend| { + const backend_io, const governor, const path, const object_store = backend; + const file = try std.Io.Dir.openFile(.cwd(), backend_io, path, .{}); + defer file.close(backend_io); + server.get_requests.store(0, .release); + try std.testing.expectEqual(read_size, try file.readPositional(backend_io, &.{&output}, 0)); - // A 503 is a server failure there: no hold, one charged retry per - // attempt. - const stats = &http.governor.stats; - try std.testing.expectEqual(@as(u64, 0), stats.holds.load(.acquire)); - try std.testing.expectEqual(@as(u64, 2), stats.server_failures.load(.acquire)); - try std.testing.expectEqual(@as(u64, 2), stats.retries.load(.acquire)); + const stats = &governor.stats; + if (object_store) { + // A 503 is `SlowDown`. One reader, two throttles in a row: each + // arms its own hold once the previous one has passed, and + // neither is charged a retry. + try std.testing.expectEqual(@as(u64, 2), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + } else { + // A 503 is a server failure: no hold, one charged retry per + // attempt. + try std.testing.expectEqual(@as(u64, 0), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.retries.load(.acquire)); + } + } try server.stop(io); } @@ -544,7 +568,7 @@ test "generic HTTP cancels a GET that receives nothing and retries it as a timeo try std.testing.expectEqual(@as(usize, 2), server.get_requests.load(.acquire)); } -test "http fails open, stat and access of an absent or denied object" { +test "http and s3 fail open, stat and access of an absent or denied object alike" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -555,72 +579,114 @@ test "http fails open, stat and access of an absent or denied object" { .{ .name = "unauthorized", .status = .unauthorized, .expected = error.PermissionDenied }, .{ .name = "forbidden", .status = .forbidden, .expected = error.PermissionDenied }, }; + // The target each backend sends for an object: HTTP its path, S3 + // `//`. + const targets = [_][]const u8{ "/", "/bucket/" }; const refusals = comptime refusals: { - var list: [cases.len]MockServer.Refusal = undefined; - for (cases, 0..) |case, i| list[i] = .{ .path = "/" ++ case.name, .status = case.status }; + var list: [cases.len * targets.len]MockServer.Refusal = undefined; + for (cases, 0..) |case, i| { + for (targets, 0..) |target, j| { + list[i * targets.len + j] = .{ .path = target ++ case.name, .status = case.status }; + } + } break :refusals list; }; var object: [32]u8 = undefined; @memset(&object, 0x6b); - var server = try MockServer.init(io, &object, .{ .refusals = &refusals }); + var server = try MockServer.init(io, &object, .{ + .objects = &.{ + .{ .path = "/bucket/object", .bytes = &object }, + }, + .refusals = &refusals, + }); defer server.deinit(io); try server.start(io); var client: std.http.Client = .{ .allocator = allocator, .io = io }; defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http }); defer http.deinit(); - const http_io = http.io(); + var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1" }); + defer s3.deinit(); - var path_buffer: [128]u8 = undefined; - const present = try server.path(&path_buffer); - try std.Io.Dir.access(.cwd(), http_io, present, .{}); - try std.testing.expectEqual(@as(u64, object.len), (try std.Io.Dir.statFile(.cwd(), http_io, present, .{})).size); - for (cases) |case| { - const path = try std.fmt.bufPrint(&path_buffer, "127.0.0.1:{d}/{s}", .{ server.port(), case.name }); - try std.testing.expectError(case.expected, std.Io.Dir.openFile(.cwd(), http_io, path, .{})); - try std.testing.expectError(case.expected, std.Io.Dir.statFile(.cwd(), http_io, path, .{})); - try std.testing.expectError(case.expected, std.Io.Dir.access(.cwd(), http_io, path, .{})); + var authority_buffer: [64]u8 = undefined; + const backends = [_]struct { std.Io, []const u8 }{ + .{ http.io(), try std.fmt.bufPrint(&authority_buffer, "{s}/", .{endpoint["http://".len..]}) }, + .{ s3.io(), "bucket/" }, + }; + for (backends) |backend| { + const backend_io, const prefix = backend; + var path_buffer: [128]u8 = undefined; + const present = try std.fmt.bufPrint(&path_buffer, "{s}object", .{prefix}); + try std.Io.Dir.access(.cwd(), backend_io, present, .{}); + try std.testing.expectEqual(@as(u64, object.len), (try std.Io.Dir.statFile(.cwd(), backend_io, present, .{})).size); + for (cases) |case| { + const path = try std.fmt.bufPrint(&path_buffer, "{s}{s}", .{ prefix, case.name }); + try std.testing.expectError(case.expected, std.Io.Dir.openFile(.cwd(), backend_io, path, .{})); + try std.testing.expectError(case.expected, std.Io.Dir.statFile(.cwd(), backend_io, path, .{})); + try std.testing.expectError(case.expected, std.Io.Dir.access(.cwd(), backend_io, path, .{})); + } } try server.stop(io); } -test "http fails a refused streaming read with InputOutput and seeks back no further than 0" { +test "http and s3 fail a refused streaming read with InputOutput and seek back no further than 0" { const allocator = std.testing.allocator; const io = std.testing.io; var object: [64]u8 = undefined; for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 5 +% 1); - // The first GET is answered 500, with no retry left. - var server = try MockServer.init(io, &object, .{ .fail_first_gets = 1 }); + // Each backend's first GET is answered 500, and none has a retry left. + var server = try MockServer.init(io, &object, .{ + .fail_first_gets = 2, + .objects = &.{ + .{ .path = "/bucket/object", .bytes = &object }, + }, + }); defer server.deinit(io); try server.start(io); var client: std.http.Client = .{ .allocator = allocator, .io = io }; defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(0) }); defer http.deinit(); - const http_io = http.io(); + var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(0) }); + defer s3.deinit(); var path_buffer: [128]u8 = undefined; - const file = try std.Io.Dir.openFile(.cwd(), http_io, try server.path(&path_buffer), .{}); - defer file.close(http_io); + const backend_ios = [_]std.Io{ http.io(), s3.io() }; + const paths = [_][]const u8{ try server.path(&path_buffer), "bucket/object" }; + var files: [backend_ios.len]std.Io.File = undefined; + var opened: usize = 0; + defer for (files[0..opened], backend_ios[0..opened]) |file, backend_io| file.close(backend_io); + for (&files, backend_ios, paths) |*file, backend_io, path| { + file.* = try std.Io.Dir.openFile(.cwd(), backend_io, path, .{}); + opened += 1; + } var head: [8]u8 = undefined; const head_slices = [_][]u8{&head}; - try std.testing.expectError(error.InputOutput, file.readStreaming(http_io, &head_slices)); - // Forward, then back by more than the position: at 0. - try http_io.vtable.fileSeekBy(http_io.userdata, file, 5); - try http_io.vtable.fileSeekBy(http_io.userdata, file, -9); - try std.testing.expectEqual(head.len, try file.readStreaming(http_io, &head_slices)); - try std.testing.expectEqualSlices(u8, object[0..head.len], &head); - // `minInt(i64)`, whose negation overflows, stops at 0 too. - try http_io.vtable.fileSeekBy(http_io.userdata, file, std.math.minInt(i64)); - try std.testing.expectEqual(head.len, try file.readStreaming(http_io, &head_slices)); - try std.testing.expectEqualSlices(u8, object[0..head.len], &head); + for (files, backend_ios) |file, backend_io| { + try std.testing.expectError(error.InputOutput, file.readStreaming(backend_io, &head_slices)); + } + for (files, backend_ios) |file, backend_io| { + // Forward, then back by more than the position: at 0. + try backend_io.vtable.fileSeekBy(backend_io.userdata, file, 5); + try backend_io.vtable.fileSeekBy(backend_io.userdata, file, -9); + try std.testing.expectEqual(head.len, try file.readStreaming(backend_io, &head_slices)); + try std.testing.expectEqualSlices(u8, object[0..head.len], &head); + // `minInt(i64)`, whose negation overflows, stops at 0 too. + try backend_io.vtable.fileSeekBy(backend_io.userdata, file, std.math.minInt(i64)); + try std.testing.expectEqual(head.len, try file.readStreaming(backend_io, &head_slices)); + try std.testing.expectEqualSlices(u8, object[0..head.len], &head); + } try server.stop(io); - try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 6), server.get_requests.load(.acquire)); } diff --git a/vfs/s3.zig b/vfs/s3.zig index d896c400..75a2d00c 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -2,7 +2,8 @@ const std = @import("std"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const VFSBase = @import("base.zig").VFSBase; const Backend = @import("base.zig").Backend; @@ -142,185 +143,26 @@ pub const AwsSigV4 = struct { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const S3 = struct { - const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - backend: *S3, - uri: std.Uri, - url: []const u8, - }; - - const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var timestamp_buf: [16]u8 = undefined; - const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch unreachable; - - const signer: AwsSigV4 = .{ - .access_key = job.batch.backend.config.access_key, - .secret_key = job.batch.backend.config.secret_key, - .region = job.batch.backend.config.region, - .service = job.batch.backend.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = signer.generateAuthHeader( - &authorization_buffer, - .GET, - job.batch.uri, - timestamp, - &.{.{ .name = "Range", .value = range_header }}, - ) catch |err| { - log.err("Failed to generate auth header: {}", .{err}); - return err; - }; - - var req = client.request(.GET, job.batch.uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - .{ .name = "Range", .value = range_header }, - }, - }) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout, .too_many_requests => .retry(), - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; - }; - - pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{ - .chunk_size = 16 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, - }; - - pub const Config = struct { + /// Without both keys, requests go unsigned. + pub const Options = struct { access_key: ?[]const u8 = null, secret_key: ?[]const u8 = null, endpoint_url: []const u8, region: []const u8, - auth_service: []const u8 = "s3", + retry: request.Retry = .{}, }; + /// The strings of the `Options` given to `init`, owned. + const Config = struct { + access_key: ?[]const u8, + secret_key: ?[]const u8, + endpoint_url: []const u8, + region: []const u8, + }; + + /// The SigV4 service every request is signed for. + const auth_service = "s3"; + const Handle = struct { pub const Type = enum { file, directory }; @@ -345,38 +187,24 @@ pub const S3 = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, - client: *std.http.Client, config: Config, - read_pool: *ParallelRead.Pool, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, - pub fn init( - allocator: std.mem.Allocator, - inner: std.Io, - http_client: *std.http.Client, - config: Config, - opts: InitOpts, - ) !S3 { - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !S3 { return .{ .allocator = allocator, .base = .init(inner), - .client = http_client, - .read_pool = read_pool, + // S3 rate limiting is `503 SlowDown` as well as `429`. + .governor = .init(inner, http_client, "s3", .throttle, options.retry), .config = .{ - .access_key = if (config.access_key) |k| try allocator.dupe(u8, k) else null, - .secret_key = if (config.secret_key) |k| try allocator.dupe(u8, k) else null, - .endpoint_url = try allocator.dupe(u8, config.endpoint_url), - .region = try allocator.dupe(u8, config.region), - .auth_service = try allocator.dupe(u8, config.auth_service), + .access_key = if (options.access_key) |k| try allocator.dupe(u8, k) else null, + .secret_key = if (options.secret_key) |k| try allocator.dupe(u8, k) else null, + .endpoint_url = try allocator.dupe(u8, options.endpoint_url), + .region = try allocator.dupe(u8, options.region), }, }; } @@ -423,13 +251,11 @@ pub const S3 = struct { .secret_key = secret_key, .endpoint_url = endpoint, .region = region, - }, .{}); + }); } pub fn deinit(self: *S3) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -456,7 +282,6 @@ pub const S3 = struct { if (self.config.secret_key) |secret_key| self.allocator.free(secret_key); self.allocator.free(self.config.endpoint_url); self.allocator.free(self.config.region); - self.allocator.free(self.config.auth_service); } pub fn io(self: *S3) std.Io { @@ -540,9 +365,9 @@ pub const S3 = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.InputOutput }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -588,11 +413,9 @@ pub const S3 = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.BadPathName => return std.Io.File.OpenError.BadPathName, - else => return std.Io.File.OpenError.Unexpected, - }; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); return .{ .inode = 0, @@ -607,18 +430,19 @@ pub const S3 = struct { }; } - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + } fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.BadPathName => return std.Io.File.OpenError.BadPathName, - else => return std.Io.File.OpenError.Unexpected, - }; - var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -644,7 +468,16 @@ pub const S3 = struct { } const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + const objects = self.listObjects(handle.uri) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.OutOfMemory => return error.SystemResources, + // Logged by `exchange`. + error.RequestFailed => return error.Unexpected, + else => { + log.err("listing {s} failed: {t}", .{ handle.uri, err }); + return error.Unexpected; + }, + }; self.dir_read_states.put(self.allocator, reader, .{ .index = 0, @@ -729,10 +562,7 @@ pub const S3 = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { @@ -742,7 +572,7 @@ pub const S3 = struct { if (relative_offset >= 0) { handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); } else { - const abs_offset: u64 = @intCast(-relative_offset); + const abs_offset: u64 = @abs(relative_offset); if (abs_offset > handle.pos) { handle.pos = 0; } else { @@ -828,49 +658,37 @@ pub const S3 = struct { .fragment = null, }; - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); + var listing: Listing = .{ .s3 = self, .uri = uri }; + var target_buffer: [256]u8 = undefined; + return request.perform([]u8, &self.governor, .{ + .target = std.fmt.bufPrint(&target_buffer, "{s}/{s}", .{ bucket, key_prefix }) catch bucket, + .key = request.authorityOf(uri), + }, &listing, Listing.attempt); + } - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; + /// The bucket listing, signed again on every attempt like every other + /// S3 request (`Signature`). + const Listing = struct { + s3: *S3, + uri: std.Uri, + signature: Signature = .{}, - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); - - var req = try self.client.request(.GET, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [2 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to list object {f}", .{uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; + fn attempt(self: *Listing, current: request.Attempt) anyerror!request.Outcome([]u8) { + return request.exchange([]u8, current, self.uri, .{ + .authorization = try self.signature.sign(self.s3, .GET, self.uri, &.{}), + .extra_headers = &self.signature.headers, + }, self, Listing.consume); } - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(self.allocator, content_len) - else - // When we don't have a content length, put a reasonable limit - try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); - } + fn consume(self: *Listing, res: *std.http.Client.Response) anyerror![]u8 { + const allocator = self.s3.allocator; + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(allocator, content_len) + else + // When we don't have a content length, put a reasonable limit + try res.reader(&.{}).allocRemaining(allocator, .limited(1024 * 1024)); + } + }; fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); @@ -913,93 +731,133 @@ pub const S3 = struct { return try results.toOwnedSlice(self.allocator); } - fn fetchSize(self: *S3, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; + fn fetchSize(self: *S3, path: []const u8) !u64 { var url_buf: [8 * 1024]u8 = undefined; - - const path = try self.resolvePath(dir, sub_path, &path_buffer); const url = try self.s3Url(path, &url_buf); const uri = std.Uri.parse(url) catch return error.BadPathName; - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); - - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .HEAD, uri, timestamp, &.{}); - - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - const res = try req.receiveHead(&redirect_buffer); - - return switch (res.head.status.class()) { - .success => res.head.content_length.?, - else => switch (res.head.status) { - .not_found => return error.FileNotFound, - else => blk: { - log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); - break :blk error.ServerError; - }, - }, - }; + var head: SizeRequest = .{ .s3 = self, .uri = uri }; + return request.perform(u64, &self.governor, .{ .target = url, .key = request.authorityOf(uri) }, &head, SizeRequest.attempt); } - fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { - const read_size = parallel_read.readSize(handle.size, offset, data); - if (read_size == 0) return 0; + /// The object's size: one signed HEAD (`ExchangeOptions.open`). S3 + /// answers a principal without `s3:ListBucket` 403 for a missing key too, + /// so a missing key may open as `PermissionDenied`. + const SizeRequest = struct { + s3: *S3, + uri: std.Uri, + signature: Signature = .{}, - var url_buf: [8 * 1024]u8 = undefined; - const url = try self.s3Url(handle.uri, &url_buf); - const uri = std.Uri.parse(url) catch return error.BadPathName; - - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .backend = self, - .uri = uri, - .url = url, - }; - - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, - }; + fn attempt(self: *SizeRequest, current: request.Attempt) anyerror!request.Outcome(u64) { + return request.exchange(u64, current, self.uri, .{ + .method = .HEAD, + .authorization = try self.signature.sign(self.s3, .HEAD, self.uri, &.{}), + .extra_headers = &self.signature.headers, + .open = true, + }, self, SizeRequest.consume); } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; + fn consume(_: *SizeRequest, res: *std.http.Client.Response) anyerror!u64 { + return res.head.content_length orelse error.MissingContentLength; + } + }; - return read_size; + fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { + var url_buf: [8 * 1024]u8 = undefined; + const url = try self.s3Url(handle.uri, &url_buf); + var signed: SignedRequest = .{ .s3 = self, .uri = std.Uri.parse(url) catch return error.BadPathName }; + return range_read.performRangeRead(&self.governor, .{ + .request = .{ .target = url, .key = request.authorityOf(signed.uri) }, + }, &signed, SignedRequest.prepare, data, offset, handle.size); } + + /// One SigV4-signed GET. The signature covers `x-amz-date` and `Range`, + /// so it is recomputed for every attempt. + const SignedRequest = struct { + s3: *S3, + uri: std.Uri, + signature: Signature = .{}, + + fn prepare(self: *SignedRequest, _: request.Attempt, range: std.http.Header) anyerror!range_read.PreparedRequest { + return .{ + .uri = self.uri, + .authorization = try self.signature.sign(self.s3, .GET, self.uri, &.{range}), + .extra_headers = &self.signature.headers, + }; + } + }; + + /// One attempt's SigV4 headers. Every request signs each attempt anew, + /// since the signature covers `x-amz-date`. The returned authorization + /// and `headers` point into this storage. + const Signature = struct { + timestamp: [16]u8 = undefined, + authorization: [512]u8 = undefined, + headers: [2]std.http.Header = undefined, + + /// The authorization over `uri` and `signed_headers` beside the two + /// headers it sets; omitted without credentials. + fn sign( + self: *Signature, + s3: *S3, + method: std.http.Method, + uri: std.Uri, + signed_headers: []const std.http.Header, + ) !std.http.Client.Request.Headers.Value { + const timestamp = try s3.getTimestamp(&self.timestamp); + const signer: AwsSigV4 = .{ + .access_key = s3.config.access_key, + .secret_key = s3.config.secret_key, + .region = s3.config.region, + .service = auth_service, + }; + const authorization = try signer.generateAuthHeader(&self.authorization, method, uri, timestamp, signed_headers); + self.headers = .{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }; + return if (authorization) |value| .{ .override = value } else .omit; + } + }; }; + +test "AwsSigV4 signs the HEAD, listing and Range GET that S3 sends" { + // AWS's example keys and date. The expected signatures are botocore's + // (S3SigV4Auth) for the same requests, with UNSIGNED-PAYLOAD as every + // request here is sent. + const signer: AwsSigV4 = .{ + .access_key = "AKIAIOSFODNN7EXAMPLE", + .secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + .region = "us-east-1", + .service = "s3", + }; + const timestamp = "20130524T000000Z"; + const credential = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, "; + var buffer: [512]u8 = undefined; + + const object = try std.Uri.parse("https://s3.amazonaws.com/examplebucket/test.txt"); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=d360d9545e619df15fd61e08c5b21c3e2219ee708af3b95ac439b0ee6fdf44b0", + (try signer.generateAuthHeader(&buffer, .HEAD, object, timestamp, &.{})).?, + ); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f483aaef7a23c6af727aeed791410b7f43f0d696542829b6bf6edc1c7cb112c4", + (try signer.generateAuthHeader(&buffer, .GET, object, timestamp, &.{.{ .name = "Range", .value = "bytes=0-9" }})).?, + ); + const listing = try std.Uri.parse("https://s3.amazonaws.com/examplebucket?delimiter=%2F&list-type=2&max-keys=1000&prefix=dir%2F"); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0a8491caa0b04a231c4ea74a87bde51ef60fbe69b496b206f4d07a4c9031188a", + (try signer.generateAuthHeader(&buffer, .GET, listing, timestamp, &.{})).?, + ); + // A custom endpoint signs its port as part of the host. + const minio = try std.Uri.parse("http://127.0.0.1:9000/models/dir/obj.bin"); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=84ff2d687bd88454372c3277af863c28ca047204936b346b80bf93456d0870d4", + (try signer.generateAuthHeader(&buffer, .HEAD, minio, timestamp, &.{})).?, + ); + + // Without both keys, requests go unsigned. + const anonymous: AwsSigV4 = .{ .access_key = "AKIAIOSFODNN7EXAMPLE", .secret_key = null, .region = "us-east-1", .service = "s3" }; + try std.testing.expectEqual(null, try anonymous.generateAuthHeader(&buffer, .HEAD, object, timestamp, &.{})); +} diff --git a/vfs/vfs.zig b/vfs/vfs.zig index 66604722..91011e23 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -1035,45 +1035,46 @@ test "creates and deletes on a local path go to the inner Io, from a path or an } test "creates and deletes on a remote path fail and write nothing anywhere" { + const allocator = std.testing.allocator; const testing_io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.writeFile(testing_io, .{ .sub_path = "existing", .data = "bytes" }); - var path_buffer: [std.fs.max_path_bytes]u8 = undefined; - const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; + var object: [16]u8 = @splat(0x2a); + var server = try MockServer.init(testing_io, &object, .{}); + defer server.deinit(testing_io); + try server.start(testing_io); - // The inner `Io` itself serves `s3`, rooted at the test's directory: a - // write that got past the VFS would land there. - var vfs: VFS = try .init(std.testing.allocator, testing_io); + var client: std.http.Client = .{ .allocator = allocator, .io = testing_io }; + defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var s3 = try S3.init(allocator, testing_io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1" }); + defer s3.deinit(); + var vfs: VFS = try .init(allocator, testing_io); defer vfs.deinit(); - try vfs.registerBackend("s3", .{ .io = testing_io }); + try vfs.registerBackend("s3", s3.backend()); const vfs_io = vfs.io(); const cwd = std.Io.Dir.cwd(); - var uri_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; - const uri = struct { - fn at(buffer: []u8, root_: []const u8, name: []const u8) ![]const u8 { - return std.fmt.bufPrint(buffer, "s3://{s}/{s}", .{ root_, name }); - } - }.at; - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFile(vfs_io, try uri(&uri_buffer, root, "key"), .{})); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.openFile(vfs_io, try uri(&uri_buffer, root, "existing"), .{ .mode = .read_write })); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFileAtomic(vfs_io, try uri(&uri_buffer, root, "key"), .{})); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDir(vfs_io, try uri(&uri_buffer, root, "dir"), .default_dir)); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPath(vfs_io, try uri(&uri_buffer, root, "a/b"))); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPathOpen(vfs_io, try uri(&uri_buffer, root, "a/b"), .{})); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteFile(vfs_io, try uri(&uri_buffer, root, "existing"))); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteDir(vfs_io, try uri(&uri_buffer, root, "dir"))); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFile(vfs_io, "s3://bucket/key", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.openFile(vfs_io, "s3://bucket/key", .{ .mode = .read_write })); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFileAtomic(vfs_io, "s3://bucket/key", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDir(vfs_io, "s3://bucket/dir", .default_dir)); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPath(vfs_io, "s3://bucket/a/b")); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPathOpen(vfs_io, "s3://bucket/a/b", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteFile(vfs_io, "s3://bucket/key")); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteDir(vfs_io, "s3://bucket/dir")); // A directory opened on the remote is no way around it. - const bucket = try cwd.openDir(vfs_io, try std.fmt.bufPrint(&uri_buffer, "s3://{s}", .{root}), .{}); + const bucket = try cwd.openDir(vfs_io, "s3://bucket", .{}); defer bucket.close(vfs_io); try std.testing.expectError(error.ReadOnlyFileSystem, bucket.createFile(vfs_io, "key", .{})); - try std.testing.expectError(error.ReadOnlyFileSystem, bucket.deleteFile(vfs_io, "existing")); + try std.testing.expectError(error.ReadOnlyFileSystem, bucket.deleteFile(vfs_io, "key")); - for ([_][]const u8{ "key", "dir", "a" }) |name| { - try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, name, .{})); + // Nothing on local disk, where the base layer would have written, and + // no request to the server. + for ([_][]const u8{ "s3:", "bucket" }) |name| { + try std.testing.expectError(error.FileNotFound, cwd.access(testing_io, name, .{})); } - try tmp.dir.access(testing_io, "existing", .{}); + try server.stop(testing_io); + try std.testing.expectEqual(0, server.head_requests.load(.acquire) + server.get_requests.load(.acquire)); } test "a name with a colon and no \"://\" is local" { From 18a9c70f20d7c42d00a6fa32fc7ebc69f0b14de4 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 4 Nov 2026 01:33:59 +0100 Subject: [PATCH 100/103] vfs/gcs: move GCS onto the governed request loop GCS's lookups, listing, reads and token refreshes now run through request.zig. GCS.init takes Options{credentials, endpoint_url, retry} instead of InitOpts, and its governor, named gcs, treats a 503 as rate limiting, since GCS answers it beside 429. A positional read is one range_read GET; the 32-worker, 16 MiB parallel_read pool goes, and InitError loses ConcurrentError with it. A token refresh, from the metadata server or the ADC token endpoint, is a governed request run under a mutex of the token's own, not the handle mutex, since it may wait out a hold or retries for minutes. Each attempt copies the bearer header into storage of its own: the slice handed out before could be rewritten by a concurrent refresh, and a token that expired during a hold is refreshed before the next try. A refresh that fails, other than by a cancel, fails the readers queued behind it, unlogged, instead of each sending it again with a retry budget of its own, which kept a load waiting out one failed refresh per reader. A refused token exchange is logged with up to 1 KiB of its body, where the endpoint says why (invalid_grant, invalid_client), as the client logged it before: request.zig's exchange gains an opt-in option for it, so the other backends log as they did. Opening, stat-ing and probing an object follow the open error contract, and access sends a HEAD where it succeeded for any path. The listing passes a cancel through and logs any other failure once. A failed positional read is Canceled or Unexpected and a failed streaming read InputOutput, each logged once. A seek saturates at 0 instead of overflowing on minInt(i64), and deinit logs the read totals at debug. The acceptance tests run the open contract, the streaming read and seek case, and the 503 hold against gs:// as well. GCS's internal Config gains the metadata server and token endpoint URLs, which the Options leave unchanged, so that unit tests can serve tokens and an object from a local server: 16 readers after an expiry share one refresh and each sends the new token; a refresh refused with 401, or failing with 500 through 2 retries, is sent once (2 and 4 token requests with the open's) and the 16 readers queued behind it fail; the ADC refresh resends the same form after a 429, which holds the backend, and after a 500, which takes one retry. Checked on real GCS anonymously, through a public bucket (vertex-model-garden-public-us): load_check of Llama-3.2-1B-Instruct (2.30 GiB, sharded on 4 CPU devices) matches every tensor's bytes, and four interleaved release runs load it in a median of 5.96 s against 5.98 s with the pool before this change, about 395 MiB/s, here in 152 requests and no retries. ls, stat, cat and safetensors work, a missing object is FileNotFound and a denied one PermissionDenied. No GCS credentials were available, so neither the metadata server nor the ADC refresh ever got a token; with a bogus ADC file, opening model.safetensors fails once, logging the token endpoint's 401 with its body (invalid_client), and a local server answering the token POST with 400 and Google's JSON error gets it logged on one line. Someone with a bucket and credentials should run load_check against it before merge. Co-Authored-By: Claude Opus 5.5 --- vfs/gcs.zig | 792 +++++++++++++++++++++-------------- vfs/http_acceptance_test.zig | 34 +- vfs/request.zig | 19 +- 3 files changed, 509 insertions(+), 336 deletions(-) diff --git a/vfs/gcs.zig b/vfs/gcs.zig index fab055e6..6717c59b 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -3,13 +3,15 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const VFSBase = @import("base.zig").VFSBase; const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/gcs"); const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; +const TokenUrl = "https://oauth2.googleapis.com/token"; /// Check if running on Google Compute Engine / Cloud Run host infrastructure, /// because metadata server probing should only happen on GCP. @@ -104,158 +106,36 @@ const Credentials = union(enum) { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const GCS = struct { - const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - backend: *GCS, - uri: std.Uri, - path: []const u8, - authorization: std.http.Client.Request.Headers.Value, - }; - - const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var req = client.request( - .GET, - job.batch.uri, - .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = job.batch.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }, - ) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout, .too_many_requests => .retry(), - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; - }; + const authorization_header_size = "Bearer ".len + OAuthToken.Size; const Token = struct { - header: []u8, + /// Guards the token. A refresh is a governed request that may wait + /// out a hold or retry for minutes, so it runs under this lock, not + /// under `GCS.mutex`, which every handle lookup takes. + mutex: std.Io.Mutex = .init, + header_storage: []u8, + header_len: usize = 0, expires_at: std.Io.Timestamp, + /// Refreshes that failed other than by a cancel. A reader that + /// queued behind one fails with it instead of running its own. + failures: std.atomic.Value(u64) = .init(0), fn expired(self: *const Token, io_: std.Io) bool { const now: std.Io.Timestamp = .now(io_, .real); return now.toSeconds() >= self.expires_at.toSeconds(); } + + fn header(self: *const Token) []const u8 { + return self.header_storage[0..self.header_len]; + } }; pub const Config = struct { credentials: ?Credentials = null, endpoint_uri: std.Uri, + /// Where tokens come from; a test points them at a local server. + metadata_uri: std.Uri = std.Uri.parse(MetadataUrl) catch unreachable, + token_uri: std.Uri = std.Uri.parse(TokenUrl) catch unreachable, }; const Handle = struct { @@ -283,48 +163,33 @@ pub const GCS = struct { allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, mutex: std.Io.Mutex = .init, - client: *std.http.Client, config: Config, token: Token, - read_pool: *ParallelRead.Pool, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, - pub const InitOpts = struct { + pub const Options = struct { credentials: ?union(enum) { json: *std.Io.Reader, metadata_server: void, } = null, endpoint_url: []const u8 = "https://storage.googleapis.com", - read_pool: parallel_read.InitOpts = .{ - .chunk_size = 16 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, + retry: request.Retry = .{}, }; pub const InitError = error{ InvalidCredentialJson, Unexpected, - } || std.mem.Allocator.Error || std.Io.ConcurrentError; + } || std.mem.Allocator.Error; - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, opts: InitOpts) InitError!GCS { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) InitError!GCS { var arena: std.heap.ArenaAllocator = .init(allocator); errdefer arena.deinit(); - - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - const config: Config = .{ - .credentials = if (opts.credentials) |creds| switch (creds) { + .credentials = if (options.credentials) |creds| switch (creds) { .json => |reader| blk: { var json_reader: std.json.Reader = .init(allocator, reader); defer json_reader.deinit(); @@ -338,11 +203,11 @@ pub const GCS = struct { }, .metadata_server => .{ .metadata_server = {} }, } else null, - .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, opts.endpoint_url)) catch return InitError.Unexpected, + .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, options.endpoint_url)) catch return InitError.Unexpected, }; const token: Token = .{ - .header = try arena.allocator().alloc(u8, "Bearer ".len + OAuthToken.Size), + .header_storage = try arena.allocator().alloc(u8, authorization_header_size), .expires_at = .zero, }; @@ -350,10 +215,10 @@ pub const GCS = struct { .allocator = allocator, .arena = arena, .base = .init(inner), - .client = http_client, .config = config, .token = token, - .read_pool = read_pool, + // GCS answers rate limiting with `503` as well as `429`. + .governor = .init(inner, http_client, "gcs", .throttle, options.retry), }; } @@ -398,50 +263,71 @@ pub const GCS = struct { return std.Io.Dir.openFile(.cwd(), io_, path, .{}) catch null; } - fn refreshMetadataServerToken(client: *std.http.Client, buffer: []u8) !?[]const u8 { - var response_writer: std.Io.Writer = .fixed(buffer); - const result = try client.fetch(.{ - .location = .{ .url = MetadataUrl }, - .method = .GET, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - }, - .extra_headers = &.{ - .{ .name = "Metadata-Flavor", .value = "Google" }, - }, - .response_writer = &response_writer, - }); - - if (result.status != .ok) { - return null; - } - - return response_writer.buffered(); + /// The metadata server's token, through the governed loop: a throttled + /// token endpoint holds the backend like any other request. + fn refreshMetadataServerToken(self: *GCS, buffer: []u8) ![]const u8 { + var token: TokenRequest = .{ + .uri = self.config.metadata_uri, + .target = "the metadata server", + .buffer = buffer, + .extra_headers = &.{.{ .name = "Metadata-Flavor", .value = "Google" }}, + }; + return request.perform([]const u8, &self.governor, token.spec(), &token, TokenRequest.attempt); } - fn refreshAuthorizedUserToken(client: *std.http.Client, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { - var response_writer: std.Io.Writer = .fixed(buffer); - const result = try client.fetch(.{ - .location = .{ .url = "https://oauth2.googleapis.com/token" }, + fn refreshAuthorizedUserToken(self: *GCS, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { + // Its own buffer: a retried attempt sends it again after the + // response was written into `buffer`. + var payload_buffer: [2 * 1024]u8 = undefined; + const payload = try std.fmt.bufPrint(&payload_buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + }); + var token: TokenRequest = .{ + .uri = self.config.token_uri, + .target = "the ADC token endpoint", + .buffer = buffer, .method = .POST, - .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), - }), - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .content_type = .{ .override = "application/x-www-form-urlencoded" }, - }, - .response_writer = &response_writer, - }); - if (result.status != .ok) { - log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); - return error.RequestFailed; - } - return response_writer.buffered(); + .payload = payload, + .content_type = .{ .override = "application/x-www-form-urlencoded" }, + }; + return request.perform([]const u8, &self.governor, token.spec(), &token, TokenRequest.attempt); } + /// One OAuth token exchange, whose body it copies into `buffer`. + const TokenRequest = struct { + uri: std.Uri, + target: []const u8, + buffer: []u8, + method: std.http.Method = .GET, + payload: ?[]u8 = null, + content_type: std.http.Client.Request.Headers.Value = .default, + extra_headers: []const std.http.Header = &.{}, + + fn spec(self: *const TokenRequest) request.RequestSpec { + return .{ .target = self.target, .key = request.authorityOf(self.uri) }; + } + + fn attempt(self: *TokenRequest, current: request.Attempt) anyerror!request.Outcome([]const u8) { + return request.exchange([]const u8, current, self.uri, .{ + .method = self.method, + .content_type = self.content_type, + .extra_headers = self.extra_headers, + .payload = self.payload, + .redirects = .follow, + // A refused exchange's body says why (`invalid_grant`, ...). + .log_refusal_body = true, + }, self, TokenRequest.consume); + } + + fn consume(self: *TokenRequest, res: *std.http.Client.Response) anyerror![]const u8 { + var writer: std.Io.Writer = .fixed(self.buffer); + _ = try res.reader(&.{}).streamRemaining(&writer); + return writer.buffered(); + } + }; + fn refreshServiceAccountToken(io_: std.Io, client: *std.http.Client, service_account: Credentials.ServiceAccount, buffer: []u8) ![]const u8 { _ = io_; _ = client; @@ -453,36 +339,44 @@ pub const GCS = struct { fn refreshToken(self: *GCS) !void { var buffer: [4 * 1024]u8 = undefined; const payload = switch (self.config.credentials.?) { - .authorized_user => |authorized_user| try refreshAuthorizedUserToken(self.client, authorized_user, &buffer), - .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.client, service_account, &buffer), - .metadata_server => try refreshMetadataServerToken(self.client, &buffer) orelse return error.RequestFailed, + .authorized_user => |authorized_user| try self.refreshAuthorizedUserToken(authorized_user, &buffer), + .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.governor.client, service_account, &buffer), + .metadata_server => try self.refreshMetadataServerToken(&buffer), }; const oauth_token = try std.json.parseFromSlice(OAuthToken, self.allocator, payload, .{ .allocate = .alloc_if_needed, .ignore_unknown_fields = true, }); defer oauth_token.deinit(); - self.token = .{ - .header = try std.fmt.bufPrint(self.token.header, "Bearer {s}", .{oauth_token.value.access_token}), - .expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))), - }; + const header = try std.fmt.bufPrint(self.token.header_storage, "Bearer {s}", .{oauth_token.value.access_token}); + self.token.header_len = header.len; + self.token.expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))); } - fn getOrRefreshToken(self: *GCS) !std.http.Client.Request.Headers.Value { + fn getOrRefreshToken(self: *GCS, authorization_buffer: []u8) !std.http.Client.Request.Headers.Value { if (self.config.credentials == null) { return .omit; } + const failures = self.token.failures.load(.monotonic); + try self.token.mutex.lock(self.base.inner); + defer self.token.mutex.unlock(self.base.inner); if (self.token.expired(self.base.inner)) { - try self.refreshToken(); + // The refresh this reader waited behind failed, and its reader + // logged why: repeating it, with a retry budget of its own, would + // only keep every other queued reader waiting. + if (self.token.failures.load(.monotonic) != failures) return error.RequestFailed; + self.refreshToken() catch |err| { + if (err != error.Canceled) _ = self.token.failures.fetchAdd(1, .monotonic); + return err; + }; } - return .{ .override = self.token.header }; + // A copy: a refresh by another reader rewrites the token's storage. + return .{ .override = try std.fmt.bufPrint(authorization_buffer, "{s}", .{self.token.header()}) }; } pub fn deinit(self: *GCS) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -588,9 +482,9 @@ pub const GCS = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.InputOutput }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -636,11 +530,9 @@ pub const GCS = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, - else => return std.Io.File.OpenError.Unexpected, - }; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); return .{ .inode = 0, @@ -655,18 +547,19 @@ pub const GCS = struct { }; } - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + } fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, - else => return std.Io.File.OpenError.Unexpected, - }; - var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -692,7 +585,16 @@ pub const GCS = struct { } const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + const objects = self.listObjects(handle.uri) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.OutOfMemory => return error.SystemResources, + // Logged by `exchange`. + error.RequestFailed => return error.Unexpected, + else => { + log.err("listing {s} failed: {t}", .{ handle.uri, err }); + return error.Unexpected; + }, + }; self.dir_read_states.put(self.allocator, reader, .{ .index = 0, @@ -777,10 +679,7 @@ pub const GCS = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { @@ -790,7 +689,7 @@ pub const GCS = struct { if (relative_offset >= 0) { handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); } else { - const abs_offset: u64 = @intCast(-relative_offset); + const abs_offset: u64 = @abs(relative_offset); if (abs_offset > handle.pos) { handle.pos = 0; } else { @@ -853,35 +752,35 @@ pub const GCS = struct { uri.path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }; uri.query = .{ .percent_encoded = query_writer.buffered() }; - var req = try self.client.request(.GET, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = try self.getOrRefreshToken(), - }, - .extra_headers = if (self.quotaProjectId()) |project| - &.{.{ .name = "x-goog-user-project", .value = project }} - else - &.{}, - }); - defer req.deinit(); + var listing: Listing = .{ .gcs = self, .uri = uri }; + return request.perform([]u8, &self.governor, .{ .target = bucket, .key = request.authorityOf(uri) }, &listing, Listing.attempt); + } - try req.sendBodiless(); + /// The bucket listing. The bearer token is copied per attempt, so one + /// that expired during a hold is refreshed before the next try. + const Listing = struct { + gcs: *GCS, + uri: std.Uri, + authorization: [authorization_header_size]u8 = undefined, - var redirect_buffer: [2 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to list object {f}", .{uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; + fn attempt(self: *Listing, current: request.Attempt) anyerror!request.Outcome([]u8) { + return request.exchange([]u8, current, self.uri, .{ + .authorization = try self.gcs.getOrRefreshToken(&self.authorization), + .extra_headers = if (self.gcs.quotaProjectId()) |project| + &.{.{ .name = "x-goog-user-project", .value = project }} + else + &.{}, + }, self, Listing.consume); } - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(self.allocator, content_len) - else - try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); - } + fn consume(self: *Listing, res: *std.http.Client.Response) anyerror![]u8 { + const allocator = self.gcs.allocator; + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(allocator, content_len) + else + try res.reader(&.{}).allocRemaining(allocator, .limited(1024 * 1024)); + } + }; fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { const bucket, const key_prefix = self.pathComponents(prefix); @@ -932,76 +831,52 @@ pub const GCS = struct { return try results.toOwnedSlice(self.allocator); } - fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; - const path = try self.resolvePath(dir, sub_path, &path_buffer); + fn fetchSize(self: *GCS, path: []const u8) !u64 { var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; const uri = try self.gcsUri(path, &uri_path_buffer); - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = try self.getOrRefreshToken(), - }, - .extra_headers = &.{}, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - const res = try req.receiveHead(&redirect_buffer); - - const size = switch (res.head.status.class()) { - .success => res.head.content_length.?, - else => switch (res.head.status) { - .not_found => return error.FileNotFound, - .unauthorized, .forbidden => return error.PermissionDenied, - else => blk: { - log.err("Failed to fetch size for {f}: {s}", .{ uri, res.head.bytes }); - break :blk error.ServerError; - }, - }, - }; - return size; + var head: SizeRequest = .{ .gcs = self, .uri = uri }; + return request.perform(u64, &self.governor, .{ .target = path, .key = request.authorityOf(uri) }, &head, SizeRequest.attempt); } - fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { - const read_size = parallel_read.readSize(handle.size, offset, data); - if (read_size == 0) return 0; + /// The object's size: one HEAD. An absent or denied object fails the + /// request (`ExchangeOptions.open`). + const SizeRequest = struct { + gcs: *GCS, + uri: std.Uri, + authorization: [authorization_header_size]u8 = undefined, - var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; - const uri = try self.gcsUri(handle.uri, &uri_path_buffer); - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .backend = self, - .uri = uri, - .path = handle.uri, - .authorization = try self.getOrRefreshToken(), - }; - - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, - }; + fn attempt(self: *SizeRequest, current: request.Attempt) anyerror!request.Outcome(u64) { + return request.exchange(u64, current, self.uri, .{ + .method = .HEAD, + .authorization = try self.gcs.getOrRefreshToken(&self.authorization), + .open = true, + }, self, SizeRequest.consume); } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; + fn consume(_: *SizeRequest, res: *std.http.Client.Response) anyerror!u64 { + return res.head.content_length orelse error.MissingContentLength; + } + }; - return read_size; + fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { + var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; + var bearer: BearerRequest = .{ .gcs = self, .uri = try self.gcsUri(handle.uri, &uri_path_buffer) }; + return range_read.performRangeRead(&self.governor, .{ + .request = .{ .target = handle.uri, .key = request.authorityOf(bearer.uri) }, + }, &bearer, BearerRequest.prepare, data, offset, handle.size); } + + /// The bearer token is copied per attempt, so one that expired during a + /// retry delay is refreshed before the next GET. + const BearerRequest = struct { + gcs: *GCS, + uri: std.Uri, + authorization: [authorization_header_size]u8 = undefined, + + fn prepare(self: *BearerRequest, _: request.Attempt, _: std.http.Header) anyerror!range_read.PreparedRequest { + return .{ .uri = self.uri, .authorization = try self.gcs.getOrRefreshToken(&self.authorization) }; + } + }; }; test "GCS object URIs are rooted at the endpoint" { @@ -1051,3 +926,272 @@ test "GCS splits bucket and object prefix" { try std.testing.expectEqualStrings("gcp-public-data-landsat", bucket); try std.testing.expectEqualStrings("LC08/01", prefix); } + +/// A token endpoint at `/token` and one object at `/bucket/object`, for the +/// token tests. The nth token request gets `answers[n]`, the last repeating; +/// a 200 grants `tok-`, k counting the grants. +const TokenServer = struct { + tcp: std.Io.net.Server, + group: std.Io.Group = .init, + object: []const u8, + answers: []const std.http.Status, + /// How long each token answer takes, so that readers queue behind a + /// refresh. + delay: std.Io.Duration = .fromMilliseconds(100), + /// The body every token request must POST form-encoded, if any. + form: ?[]const u8 = null, + token_requests: std.atomic.Value(usize) = .init(0), + granted: std.atomic.Value(usize) = .init(0), + object_requests: std.atomic.Value(usize) = .init(0), + /// Object requests without the last token granted. + stale_requests: std.atomic.Value(usize) = .init(0), + /// Token requests that did not POST `form`. + bad_forms: std.atomic.Value(usize) = .init(0), + first_error: stdx.FirstError(anyerror) = .{}, + + fn init(io: std.Io, object: []const u8, answers: []const std.http.Status) !TokenServer { + const address: std.Io.net.IpAddress = .{ .ip4 = .loopback(0) }; + return .{ .tcp = try address.listen(io, .{ .reuse_address = true }), .object = object, .answers = answers }; + } + + fn deinit(self: *TokenServer, io: std.Io) void { + self.group.cancel(io); + self.tcp.deinit(io); + } + + fn run(self: *TokenServer, io: std.Io) std.Io.Cancelable!void { + var connections: std.Io.Group = .init; + defer connections.cancel(io); + while (true) { + const stream = self.tcp.accept(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + _ = self.first_error.set(err); + return; + }, + }; + connections.concurrent(io, onConnection, .{ self, stream, io }) catch |err| { + stream.close(io); + _ = self.first_error.set(err); + return; + }; + } + } + + fn onConnection(self: *TokenServer, stream: std.Io.net.Stream, io: std.Io) std.Io.Cancelable!void { + defer stream.close(io); + self.answer(stream, io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => _ = self.first_error.set(err), + }; + } + + fn answer(self: *TokenServer, stream: std.Io.net.Stream, io: std.Io) !void { + var read_buffer: [8 * 1024]u8 = undefined; + var reader = stream.reader(io, &read_buffer); + var write_buffer: [8 * 1024]u8 = undefined; + var writer = stream.writer(io, &write_buffer); + var server: std.http.Server = .init(&reader.interface, &writer.interface); + var req = try server.receiveHead(); + + if (std.mem.eql(u8, req.head.target, "/token")) { + const n = self.token_requests.fetchAdd(1, .acq_rel); + if (self.form) |form| { + const content_type = req.head.content_type orelse ""; + const form_encoded = req.head.method == .POST and std.mem.eql(u8, content_type, "application/x-www-form-urlencoded"); + var body_buffer: [512]u8 = undefined; + var body: std.Io.Writer = .fixed(&body_buffer); + _ = try req.readerExpectNone(&.{}).streamRemaining(&body); + if (!form_encoded or !std.mem.eql(u8, body.buffered(), form)) _ = self.bad_forms.fetchAdd(1, .acq_rel); + } + try io.sleep(self.delay, .awake); + const status = self.answers[@min(n, self.answers.len - 1)]; + if (status != .ok) return req.respond("{\"error\":\"invalid_grant\"}", .{ .status = status, .keep_alive = false }); + var token_buffer: [64]u8 = undefined; + const token = try std.fmt.bufPrint(&token_buffer, "{{\"access_token\":\"tok-{d}\",\"expires_in\":3600}}", .{self.granted.fetchAdd(1, .acq_rel) + 1}); + return req.respond(token, .{ .keep_alive = false }); + } + + if (!std.mem.eql(u8, req.head.target, "/bucket/object?alt=media")) return error.UnexpectedTarget; + _ = self.object_requests.fetchAdd(1, .acq_rel); + var authorization: []const u8 = ""; + var range: ?[]const u8 = null; + var headers = req.iterateHeaders(); + while (headers.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "authorization")) authorization = header.value; + if (std.ascii.eqlIgnoreCase(header.name, "range")) range = header.value; + } + var expected_buffer: [32]u8 = undefined; + const expected = try std.fmt.bufPrint(&expected_buffer, "Bearer tok-{d}", .{self.granted.load(.acquire)}); + if (!std.mem.eql(u8, authorization, expected)) _ = self.stale_requests.fetchAdd(1, .acq_rel); + if (req.head.method == .HEAD) return req.respond(self.object, .{ .keep_alive = false }); + + const spec = (range orelse return error.MissingRange)["bytes=".len..]; + const dash = std.mem.indexOfScalar(u8, spec, '-') orelse return error.InvalidRange; + const first = try std.fmt.parseInt(usize, spec[0..dash], 10); + const last = try std.fmt.parseInt(usize, spec[dash + 1 ..], 10); + var content_range_buffer: [64]u8 = undefined; + return req.respond(self.object[first .. last + 1], .{ + .status = .partial_content, + .keep_alive = false, + .extra_headers = &.{.{ + .name = "Content-Range", + .value = try std.fmt.bufPrint(&content_range_buffer, "bytes {d}-{d}/{d}", .{ first, last, self.object.len }), + }}, + }); + } +}; + +/// A GCS whose tokens and object come from a `TokenServer`. It must not +/// move once set up. +const TokenFixture = struct { + object: [4096]u8 = undefined, + server: TokenServer = undefined, + client: std.http.Client = undefined, + gcs: GCS = undefined, + endpoint_buffer: [32]u8 = undefined, + token_url_buffer: [48]u8 = undefined, + + fn setUp( + self: *TokenFixture, + credentials: @FieldType(GCS.Options, "credentials"), + answers: []const std.http.Status, + max_retries: usize, + ) !void { + const io = std.testing.io; + for (&self.object, 0..) |*byte, i| byte.* = @truncate(i *% 7 +% 3); + self.server = try .init(io, &self.object, answers); + errdefer self.server.deinit(io); + try self.server.group.concurrent(io, TokenServer.run, .{ &self.server, io }); + self.client = .{ .allocator = std.testing.allocator, .io = io }; + errdefer self.client.deinit(); + const port = self.server.tcp.socket.address.getPort(); + const token_uri: std.Uri = try .parse(try std.fmt.bufPrint(&self.token_url_buffer, "http://127.0.0.1:{d}/token", .{port})); + self.gcs = try .init(std.testing.allocator, io, &self.client, .{ + .credentials = credentials, + .endpoint_url = try std.fmt.bufPrint(&self.endpoint_buffer, "http://127.0.0.1:{d}", .{port}), + .retry = .{ + .max_retries = max_retries, + .initial_delay = .fromMilliseconds(5), + .max_delay = .fromMilliseconds(5), + .max_hold = .fromMilliseconds(20), + }, + }); + self.gcs.config.metadata_uri = token_uri; + self.gcs.config.token_uri = token_uri; + } + + fn tearDown(self: *TokenFixture) void { + self.gcs.deinit(); + self.client.deinit(); + self.server.deinit(std.testing.io); + } + + /// Stops the server and returns the first error it met. + fn stop(self: *TokenFixture) !void { + self.server.group.cancel(std.testing.io); + if (self.server.first_error.get()) |err| return err; + } + + /// `readers` concurrent reads of 256 bytes each, started together; + /// returns how many failed. Every read that succeeds must match. + fn readConcurrently(self: *TokenFixture, file: std.Io.File, comptime readers: usize) !usize { + const io = std.testing.io; + const Reader = struct { + fn run(gcs_io: std.Io, file_: std.Io.File, output: []u8, offset: u64, go: *std.Io.Event, result: *anyerror!usize) void { + go.wait(gcs_io) catch |err| { + result.* = err; + return; + }; + result.* = file_.readPositional(gcs_io, &.{output}, offset); + } + }; + var outputs: [readers][256]u8 = undefined; + var results: [readers]anyerror!usize = undefined; + var go: std.Io.Event = .unset; + var group: std.Io.Group = .init; + defer group.cancel(io); + for (&outputs, &results, 0..) |*output, *result, i| { + try group.concurrent(io, Reader.run, .{ self.gcs.io(), file, output, i * output.len, &go, result }); + } + go.set(io); + try group.await(io); + + var failed: usize = 0; + for (&outputs, results, 0..) |*output, result, i| { + const n = result catch { + failed += 1; + continue; + }; + try std.testing.expectEqual(output.len, n); + try std.testing.expectEqualSlices(u8, self.object[i * output.len ..][0..output.len], output); + } + return failed; + } +}; + +test "GCS refreshes an expired token once for concurrent readers, and each sends the new one" { + var fixture: TokenFixture = .{}; + try fixture.setUp(.{ .metadata_server = {} }, &.{.ok}, 0); + defer fixture.tearDown(); + const server = &fixture.server; + const gcs_io = fixture.gcs.io(); + const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); + defer file.close(gcs_io); + try std.testing.expectEqual(1, server.token_requests.load(.acquire)); + + fixture.gcs.token.expires_at = .zero; + try std.testing.expectEqual(0, try fixture.readConcurrently(file, 16)); + try fixture.stop(); + try std.testing.expectEqual(2, server.token_requests.load(.acquire)); + try std.testing.expectEqual(17, server.object_requests.load(.acquire)); + try std.testing.expectEqual(0, server.stale_requests.load(.acquire)); +} + +test "GCS fails the readers queued behind a failed token refresh without repeating it" { + const cases = [_]struct { status: std.http.Status, max_retries: usize, token_requests: usize }{ + // The open's request, then one refused refresh. + .{ .status = .unauthorized, .max_retries = 0, .token_requests = 2 }, + // The open's request, then one refresh and its 2 retries. + .{ .status = .internal_server_error, .max_retries = 2, .token_requests = 4 }, + }; + for (cases) |case| { + var fixture: TokenFixture = .{}; + try fixture.setUp(.{ .metadata_server = {} }, &.{ .ok, case.status }, case.max_retries); + defer fixture.tearDown(); + const server = &fixture.server; + const gcs_io = fixture.gcs.io(); + const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); + defer file.close(gcs_io); + + fixture.gcs.token.expires_at = .zero; + try std.testing.expectEqual(16, try fixture.readConcurrently(file, 16)); + try fixture.stop(); + try std.testing.expectEqual(case.token_requests, server.token_requests.load(.acquire)); + try std.testing.expectEqual(1, server.object_requests.load(.acquire)); + } +} + +test "GCS resends the ADC refresh form on a retry, and a throttled token endpoint holds the backend" { + var json: std.Io.Reader = .fixed( + \\{"type":"authorized_user","client_id":"cid.apps.googleusercontent.com","client_secret":"GOCSPX-sec","refresh_token":"1//0abc-def_ghi","account":"","universe_domain":"googleapis.com"} + ); + var fixture: TokenFixture = .{}; + try fixture.setUp(.{ .json = &json }, &.{ .too_many_requests, .internal_server_error, .ok }, 2); + defer fixture.tearDown(); + const server = &fixture.server; + server.delay = .zero; + server.form = "grant_type=refresh_token&client_id=cid.apps.googleusercontent.com&client_secret=GOCSPX-sec&refresh_token=1//0abc-def_ghi"; + const gcs_io = fixture.gcs.io(); + const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); + defer file.close(gcs_io); + + try std.testing.expectEqual(0, try fixture.readConcurrently(file, 4)); + try fixture.stop(); + const stats = &fixture.gcs.governor.stats; + try std.testing.expectEqual(1, stats.holds.load(.acquire)); + try std.testing.expectEqual(1, stats.retries.load(.acquire)); + try std.testing.expectEqual(3, server.token_requests.load(.acquire)); + try std.testing.expectEqual(0, server.bad_forms.load(.acquire)); + try std.testing.expectEqual(0, server.stale_requests.load(.acquire)); +} diff --git a/vfs/http_acceptance_test.zig b/vfs/http_acceptance_test.zig index eca825fc..a09464a5 100644 --- a/vfs/http_acceptance_test.zig +++ b/vfs/http_acceptance_test.zig @@ -1,5 +1,6 @@ const std = @import("std"); +const GCS = @import("gcs.zig").GCS; const HTTP = @import("http.zig").HTTP; const MockServer = @import("mock_server.zig").MockServer; const range_read = @import("range_read.zig"); @@ -412,6 +413,7 @@ test "a 503 holds an object store and only retries a plain HTTP server" { .throttle = .{ .first_gets = 2, .status = .service_unavailable }, .objects = &.{ .{ .path = "/bucket/object", .bytes = &object }, + .{ .path = "/bucket/object?alt=media", .bytes = &object }, }, }); defer server.deinit(io); @@ -423,12 +425,15 @@ test "a 503 holds an object store and only retries a plain HTTP server" { const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(5) }); defer s3.deinit(); + var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint, .retry = instantRetries(5) }); + defer gcs.deinit(); var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(5) }); defer http.deinit(); var path_buffer: [128]u8 = undefined; const backends = [_]struct { std.Io, *const vfs_request.Governor, []const u8, bool }{ .{ s3.io(), &s3.governor, "bucket/object", true }, + .{ gcs.io(), &gcs.governor, "bucket/object", true }, .{ http.io(), &http.governor, try server.path(&path_buffer), false }, }; for (backends) |backend| { @@ -440,8 +445,8 @@ test "a 503 holds an object store and only retries a plain HTTP server" { const stats = &governor.stats; if (object_store) { - // A 503 is `SlowDown`. One reader, two throttles in a row: each - // arms its own hold once the previous one has passed, and + // A 503 is rate limiting. One reader, two throttles in a row: + // each arms its own hold once the previous one has passed, and // neither is charged a retry. try std.testing.expectEqual(@as(u64, 2), stats.holds.load(.acquire)); try std.testing.expectEqual(@as(u64, 2), stats.throttles.load(.acquire)); @@ -568,7 +573,7 @@ test "generic HTTP cancels a GET that receives nothing and retries it as a timeo try std.testing.expectEqual(@as(usize, 2), server.get_requests.load(.acquire)); } -test "http and s3 fail open, stat and access of an absent or denied object alike" { +test "http, s3 and gs fail open, stat and access of an absent or denied object alike" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -580,13 +585,13 @@ test "http and s3 fail open, stat and access of an absent or denied object alike .{ .name = "forbidden", .status = .forbidden, .expected = error.PermissionDenied }, }; // The target each backend sends for an object: HTTP its path, S3 - // `//`. - const targets = [_][]const u8{ "/", "/bucket/" }; + // `//`, GCS the same with `?alt=media`. + const targets = [_]struct { []const u8, []const u8 }{ .{ "/", "" }, .{ "/bucket/", "" }, .{ "/bucket/", "?alt=media" } }; const refusals = comptime refusals: { var list: [cases.len * targets.len]MockServer.Refusal = undefined; for (cases, 0..) |case, i| { for (targets, 0..) |target, j| { - list[i * targets.len + j] = .{ .path = target ++ case.name, .status = case.status }; + list[i * targets.len + j] = .{ .path = target[0] ++ case.name ++ target[1], .status = case.status }; } } break :refusals list; @@ -597,6 +602,7 @@ test "http and s3 fail open, stat and access of an absent or denied object alike var server = try MockServer.init(io, &object, .{ .objects = &.{ .{ .path = "/bucket/object", .bytes = &object }, + .{ .path = "/bucket/object?alt=media", .bytes = &object }, }, .refusals = &refusals, }); @@ -611,11 +617,14 @@ test "http and s3 fail open, stat and access of an absent or denied object alike defer http.deinit(); var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1" }); defer s3.deinit(); + var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint }); + defer gcs.deinit(); var authority_buffer: [64]u8 = undefined; const backends = [_]struct { std.Io, []const u8 }{ .{ http.io(), try std.fmt.bufPrint(&authority_buffer, "{s}/", .{endpoint["http://".len..]}) }, .{ s3.io(), "bucket/" }, + .{ gcs.io(), "bucket/" }, }; for (backends) |backend| { const backend_io, const prefix = backend; @@ -634,7 +643,7 @@ test "http and s3 fail open, stat and access of an absent or denied object alike try server.stop(io); } -test "http and s3 fail a refused streaming read with InputOutput and seek back no further than 0" { +test "http, s3 and gs fail a refused streaming read with InputOutput and seek back no further than 0" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -642,9 +651,10 @@ test "http and s3 fail a refused streaming read with InputOutput and seek back n for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 5 +% 1); // Each backend's first GET is answered 500, and none has a retry left. var server = try MockServer.init(io, &object, .{ - .fail_first_gets = 2, + .fail_first_gets = 3, .objects = &.{ .{ .path = "/bucket/object", .bytes = &object }, + .{ .path = "/bucket/object?alt=media", .bytes = &object }, }, }); defer server.deinit(io); @@ -658,10 +668,12 @@ test "http and s3 fail a refused streaming read with InputOutput and seek back n defer http.deinit(); var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(0) }); defer s3.deinit(); + var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint, .retry = instantRetries(0) }); + defer gcs.deinit(); var path_buffer: [128]u8 = undefined; - const backend_ios = [_]std.Io{ http.io(), s3.io() }; - const paths = [_][]const u8{ try server.path(&path_buffer), "bucket/object" }; + const backend_ios = [_]std.Io{ http.io(), s3.io(), gcs.io() }; + const paths = [_][]const u8{ try server.path(&path_buffer), "bucket/object", "bucket/object" }; var files: [backend_ios.len]std.Io.File = undefined; var opened: usize = 0; defer for (files[0..opened], backend_ios[0..opened]) |file, backend_io| file.close(backend_io); @@ -688,5 +700,5 @@ test "http and s3 fail a refused streaming read with InputOutput and seek back n } try server.stop(io); - try std.testing.expectEqual(@as(usize, 6), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 9), server.get_requests.load(.acquire)); } diff --git a/vfs/request.zig b/vfs/request.zig index 6aa38f24..9365d460 100644 --- a/vfs/request.zig +++ b/vfs/request.zig @@ -494,6 +494,9 @@ pub const ExchangeOptions = struct { /// request with its `OpenError` before `consume`, unlogged. open: bool = false, redirects: Redirects = .forbid, + /// A refusal that fails the request is logged with up to 1 KiB of its + /// body, where an OAuth endpoint says why, instead of with its head. + log_refusal_body: bool = false, }; /// One HTTP exchange of `attempt`'s request: send, receive the head, @@ -623,7 +626,21 @@ fn roundTrip( return .{ .retry = .{ .failure = .transient, .status = res.head.status } }; } const failure = classifyStatus(res.head.status, attempt.governor.unavailable) orelse { - log.err("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); + if (!options.log_refusal_body) { + log.err("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); + return error.RequestFailed; + } + // Reading the body invalidates the head's strings. + const status = res.head.status; + var body_buffer: [1024]u8 = undefined; + var body: std.Io.Writer = .fixed(&body_buffer); + // A longer body is cut, one that fails is logged as it arrived. + _ = res.reader(&.{}).streamRemaining(&body) catch { + if (canceled(&req)) return error.Canceled; + }; + // One line: an OAuth endpoint spreads its JSON over several. + std.mem.replaceScalar(u8, body.buffered(), '\n', ' '); + log.err("{s}: {s} answered {d}: {s}", .{ attempt.governor.name, spec.target, @backingInt(status), body.buffered() }); return error.RequestFailed; }; log.warn("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); From dbcc3f69034c2f74f2239d77252cbeba209a906b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 4 Nov 2026 01:34:15 +0100 Subject: [PATCH 101/103] vfs/hf: move HF onto the governed request loop HF's repository listing, download-URL lookups and reads now run through request.zig, under a governor named hf for which a 503 is a server failure. HF.init takes Options{token, retry} in place of a token and InitOpts; auto is unchanged. A file's download URL is resolved once at open with HeadChain (at most 8 hops) and kept on its handle, so a read is one Range GET to where the Hub's redirect chain ends, not a redirect chain per 32 MiB chunk. The 32-worker parallel_read pool goes: a read's width is its caller's. The token is sent to https://huggingface.co only: never to the CDN a resolve URL redirects to, a look-alike host or plain http, as a unit test checks. A 401 or 403 on a data GET, as a signed URL that expired during a long hold answers, re-resolves the URL once and stores it on the handle for later reads. The listing is fetched under its own mutex, so reads of open files go on while it waits; the cache was read without a lock and the fetch held the handle mutex. Also: stat of a path absent from the listing is FileNotFound, not IsDir, and opening a directory of the listing is IsDir, as its stat is, without a request; a listing without Content-Length fails instead of panicking, and one that fails partway frees the nodes it built. Opening, stat-ing and probing follow the open error contract (a gated repository without a token opens as PermissionDenied); a failed positional read is Canceled or Unexpected and a failed streaming read InputOutput, each logged once. A seek saturates, and deinit logs the read totals at debug. Validated on CPU against the pool, both on the new loader, interleaved, sharded on 4 devices. load_check of hf://LiquidAI/LFM2.5-1.2B-Thinking loads 2.18 GiB in 2.71 s against 2.68 s with the pool (medians of 5, about 830 MiB/s, this host's network), with 76 requests and no retries; with every tensor checked back, the run takes 35 s instead of 50 s, since a check read no longer walks the redirect chain. Of hf://Qwen/Qwen3.5-9B it loads 17.98 GiB in 19.41-19.52 s against 19.36-19.37 s with the pool (about 950 MiB/s), with 603 requests and no retries, and a run checking every 8th tensor and the largest matched. With the VFS's debug log on, examples/io cat of hf://Qwen/Qwen3.5-9B/config.json takes 4 requests and safetensors 18, listing the same tensors as before; a missing file is FileNotFound. Not run: a gated repository with a token (none here), a re-resolve after a real expiry (the mock server cannot stand in for huggingface.co), and the B70 runs. Co-Authored-By: Claude Opus 5.5 --- vfs/hf.zig | 527 ++++++++++++++++++++++++----------------------------- 1 file changed, 241 insertions(+), 286 deletions(-) diff --git a/vfs/hf.zig b/vfs/hf.zig index d1a61a6d..2989d61f 100644 --- a/vfs/hf.zig +++ b/vfs/hf.zig @@ -2,161 +2,13 @@ const std = @import("std"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const VFSBase = @import("base.zig").VFSBase; const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/hf"); -const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - uri: std.Uri, - url: []const u8, - authorization: std.http.Client.Request.Headers.Value, - }; - - pub const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var req = client.request(.GET, job.batch.uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = job.batch.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout => .retry(), - .too_many_requests => b: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { - var parts = std.mem.splitScalar(u8, header.value, ';'); - while (parts.next()) |part| { - const p = std.mem.trim(u8, part, " \t"); - if (std.mem.startsWith(u8, p, "t=")) { - const seconds = std.fmt.parseInt(u32, p[2..], 10) catch continue; - break :b .{ .retry_after = .fromSeconds(seconds) }; - } - } - } - } - break :b .retry(); - }, - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; -}; - pub const API = struct { const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}?expand=false&recursive=true&limit=1000"; const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; @@ -201,16 +53,10 @@ const RepoKey = struct { }; pub const HF = struct { - pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{ - // Chunk size needs to be big enough to avoid hitting the rate limits of HF. - .chunk_size = 32 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, + pub const Options = struct { + /// Sent as a bearer token to the Hub itself, never to its CDN. + token: ?[]const u8 = null, + retry: request.Retry = .{}, }; pub const Repo = struct { @@ -244,13 +90,17 @@ pub const HF = struct { type: Type, uri: []const u8, + download_url: ?[]const u8, pos: u64, size: u64, - pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, size: u64) !Handle { + pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, download_url: ?[]const u8, size: u64) !Handle { + const owned_uri = try allocator.dupe(u8, uri); + errdefer allocator.free(owned_uri); return .{ .type = handle_type, - .uri = try allocator.dupe(u8, uri), + .uri = owned_uri, + .download_url = if (download_url) |url| try allocator.dupe(u8, url) else null, .pos = 0, .size = size, }; @@ -258,39 +108,37 @@ pub const HF = struct { pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { allocator.free(self.uri); + if (self.download_url) |url| allocator.free(url); } }; allocator: std.mem.Allocator, + /// Guards the handle table and the handles' download URLs. mutex: std.Io.Mutex = .init, - client: *std.http.Client, authorization: std.http.Client.Request.Headers.Value, - read_pool: *ParallelRead.Pool, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, + /// Guards `trees`. A tree fetch is a governed request that may wait out + /// a hold or retry for minutes, so it runs under this lock, not under + /// `mutex`, which every read takes. + trees_mutex: std.Io.Mutex = .init, trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8, opts: InitOpts) !HF { - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !HF { var self: HF = .{ .allocator = allocator, .base = .init(inner), - .client = http_client, - .authorization = if (hf_token) |token| blk: { + .authorization = if (options.token) |token| blk: { break :blk .{ .override = try std.fmt.allocPrint(allocator, "Bearer {s}", .{std.mem.trim(u8, token, " \t\n\r")}), }; } else blk: { break :blk .default; }, - .read_pool = read_pool, + .governor = .init(inner, http_client, "hf", .server_failure, options.retry), }; errdefer switch (self.authorization) { .default, .omit => {}, @@ -322,13 +170,11 @@ pub const HF = struct { if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); - return init(allocator, inner, http_client, hf_token, .{}); + return init(allocator, inner, http_client, .{ .token = hf_token }); } pub fn deinit(self: *HF) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -440,39 +286,36 @@ pub const HF = struct { return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); } - fn getOrFetchTree(self: *HF, repo: Repo) !*std.ArrayList(TreeNode) { + /// The repository's listing, fetched once. The lock is held across the + /// fetch, so every open, stat or listing of this backend waits for it, + /// cached or not. The map is read and written under `trees_mutex`, since + /// an insert may move its entries; the nodes themselves never move once + /// fetched, so the returned slice stays valid without the lock. + fn getOrFetchTree(self: *HF, repo: Repo) ![]const TreeNode { const key = repo.toKey(); var cache_key_buffer: [512]u8 = undefined; - const cache_key_lookup = try key.format(&cache_key_buffer); - if (self.trees.getPtr(cache_key_lookup)) |tree_ptr| return tree_ptr; + try self.trees_mutex.lock(self.base.inner); + defer self.trees_mutex.unlock(self.base.inner); + if (self.trees.get(cache_key_lookup)) |tree| return tree.items; + + // Room first: once fetched, the listing cannot fail to be kept. + try self.trees.ensureUnusedCapacity(self.allocator, 1); const cache_key = try key.allocFormat(self.allocator); errdefer self.allocator.free(cache_key); - - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - const gop = try self.trees.getOrPut(self.allocator, cache_key); - errdefer _ = self.trees.remove(cache_key); - - if (gop.found_existing) { - self.allocator.free(cache_key); - return gop.value_ptr; - } - - const tree_root = try self.fetchTreeFromAPI(repo); - gop.value_ptr.* = tree_root; - - return gop.value_ptr; + const tree = try self.fetchTreeFromAPI(repo); + self.trees.putAssumeCapacityNoClobber(cache_key, tree); + return tree.items; } + /// The repository listing, through the governed loop like every other + /// request. It runs under `trees_mutex` from `getOrFetchTree`, never + /// under `mutex`, so reads of open files go on while it waits. fn fetchTreeFromAPI(self: *HF, repo: Repo) !std.ArrayList(TreeNode) { var url_buffer: [8 * 1024]u8 = undefined; - var redirect_buffer: [8 * 1024]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, API.TREE_URL_TEMPLATE, .{ .repo = repo.repo, .model = repo.model, @@ -482,40 +325,64 @@ pub const HF = struct { const uri = try std.Uri.parse(url); log.info("Fetching from HF: {s}", .{url}); - var req = try self.client.request(.GET, uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = self.authorization, - }, - }); - defer req.deinit(); + var fetch: TreeFetch = .{ .hf = self, .uri = uri }; + return request.perform(std.ArrayList(TreeNode), &self.governor, .{ .target = url, .key = request.authorityOf(uri) }, &fetch, TreeFetch.attempt); + } - try req.sendBodiless(); + const TreeFetch = struct { + hf: *HF, + uri: std.Uri, - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to fetch tree: status={}", .{res.head.status}); - return error.RequestFailed; + fn attempt(self: *TreeFetch, current: request.Attempt) anyerror!request.Outcome(std.ArrayList(TreeNode)) { + return request.exchange(std.ArrayList(TreeNode), current, self.uri, .{ + .authorization = self.hf.authorization, + .open = true, + .redirects = .follow, + }, self, TreeFetch.consume); } - const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length.?); - defer self.allocator.free(body); + fn consume(self: *TreeFetch, res: *std.http.Client.Response) anyerror!std.ArrayList(TreeNode) { + const allocator = self.hf.allocator; + const body = try res.reader(&.{}).readAlloc(allocator, res.head.content_length orelse return error.MissingContentLength); + defer allocator.free(body); - const parsed = try std.json.parseFromSlice( - []API.Tree, - self.allocator, - body, - .{ .ignore_unknown_fields = true }, - ); - defer parsed.deinit(); + const parsed = try std.json.parseFromSlice( + []API.Tree, + allocator, + body, + .{ .ignore_unknown_fields = true }, + ); + defer parsed.deinit(); - var tree_root: std.ArrayList(TreeNode) = .empty; - for (parsed.value) |item| { - try insertTreeNode(self.allocator, &tree_root, item); + var tree_root: std.ArrayList(TreeNode) = .empty; + errdefer { + for (tree_root.items) |*node| node.deinit(allocator); + tree_root.deinit(allocator); + } + for (parsed.value) |item| { + try insertTreeNode(allocator, &tree_root, item); + } + return tree_root; } + }; - return tree_root; + /// The URL the file's bytes are read from, copied into `out_buffer`: + /// where the Hub's redirect chain ends, a signed CDN URL for an LFS file. + fn resolveDownloadUrl(self: *HF, repo: Repo, out_buffer: []u8) ![]const u8 { + var url_buffer: [8 * 1024]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); + // Logs name the file: past the first hop the URL is a signed one, + // which is a credential. + var target_buffer: [1024]u8 = undefined; + const target = std.fmt.bufPrint(&target_buffer, "{s}/{s}@{s}/{s}", .{ repo.repo, repo.model, repo.rev, repo.path }) catch repo.path; + var chain: request.HeadChain = .{}; + const end = try chain.resolve(&self.governor, target, url, 8, self, hubAuthorization); + return std.fmt.bufPrint(out_buffer, "{s}", .{end.url}); + } + + /// Credentials only travel to the Hub itself, never to its CDN. + fn hubAuthorization(self: *HF, url: []const u8) std.http.Client.Request.Headers.Value { + return if (std.mem.startsWith(u8, url, "https://huggingface.co/")) self.authorization else .omit; } fn findDirChildren(tree_items: []const TreeNode, dir_path: []const u8) ?[]const TreeNode { @@ -581,16 +448,16 @@ pub const HF = struct { return null; } - fn getSizeFromTree(tree: *std.ArrayList(TreeNode), path: []const u8) ?u64 { + fn getSizeFromTree(tree: []const TreeNode, path: []const u8) ?u64 { if (path.len == 0) { var total: u64 = 0; - for (tree.items) |*node| { + for (tree) |*node| { if (node.kind == .file) total += node.size; } return total; } - const node = findNode(tree.items, path) orelse return null; + const node = findNode(tree, path) orelse return null; return node.size; } @@ -651,9 +518,9 @@ pub const HF = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.InputOutput }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -676,11 +543,11 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.OpenError.BadPathName; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.OpenError.Unexpected; + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.OpenError.FileNotFound; const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .directory, full_path, size) catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, full_path, null, size) catch return std.Io.Dir.OpenError.Unexpected; return .{ .handle = @intCast(idx) }; } @@ -709,16 +576,16 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.StatFileError.Unexpected; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.StatFileError.Unexpected; - const node = findNode(tree.items, repo.path) orelse return std.Io.Dir.StatFileError.IsDir; - + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); + // The repository itself is a directory. + if (repo.path.len == 0) return std.Io.Dir.StatFileError.IsDir; + const node = findNode(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; if (node.kind == .directory) return std.Io.Dir.StatFileError.IsDir; - const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; return .{ .inode = @intCast(0), .nlink = 0, - .size = size, + .size = node.size, .permissions = .fromMode(0o444), .kind = .file, .atime = null, @@ -735,11 +602,11 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.AccessError.FileNotFound; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.AccessError.Unexpected; + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); if (repo.path.len == 0) return; - const node = findNode(tree.items, repo.path); + const node = findNode(tree, repo.path); if (node == null) return std.Io.Dir.AccessError.FileNotFound; } @@ -750,11 +617,17 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.File.OpenError.BadPathName; - const tree = self.getOrFetchTree(repo) catch return std.Io.File.OpenError.Unexpected; - const size = getSizeFromTree(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); + // A directory, the repository itself included, has no download URL. + if (repo.path.len == 0) return std.Io.File.OpenError.IsDir; + const node = findNode(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; + if (node.kind == .directory) return std.Io.File.OpenError.IsDir; + var download_url_buffer: [16 * 1024]u8 = undefined; + // Logged by the file's path: its signed URL would be a credential. + const download_url = self.resolveDownloadUrl(repo, &download_url_buffer) catch |err| return self.governor.openFailure(full_path, err); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .file, full_path, size) catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, full_path, download_url, node.size) catch return std.Io.File.OpenError.Unexpected; return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; } @@ -776,9 +649,12 @@ pub const HF = struct { const handle = self.getDirHandle(reader.dir); const repo = Repo.parse(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.Reader.Error.Unexpected; + const tree = self.getOrFetchTree(repo) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return std.Io.Dir.Reader.Error.Unexpected, + }; - const children = findDirChildren(tree.items, repo.path) orelse &.{}; + const children = findDirChildren(tree, repo.path) orelse &.{}; self.dir_read_states.put(self.allocator, reader, .{ .children = children, @@ -856,20 +732,18 @@ pub const HF = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); + // Saturates at both ends; `@abs` takes `minInt(i64)` too. handle.pos = if (relative_offset >= 0) - handle.pos + @as(u64, @intCast(relative_offset)) + handle.pos +| @abs(relative_offset) else - handle.pos - @as(u64, @intCast(-relative_offset)); + handle.pos -| @abs(relative_offset); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { @@ -888,44 +762,125 @@ pub const HF = struct { fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { if (offset >= handle.size) return 0; - var read_size: usize = 0; - for (data) |buf| read_size += buf.len; - read_size = @intCast(@min(handle.size - offset, read_size)); - if (read_size == 0) return 0; - - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - const repo: Repo = try .parse(handle.uri); - var url_buffer: [8 * 1024]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); - const uri: std.Uri = try .parse(url); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .uri = uri, - .url = url, - .authorization = self.authorization, + // Another read of the file may replace the handle's URL + // (`DataRequest.reresolve`): each read sends its own copy. + const url = url: { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + break :url try self.allocator.dupe(u8, handle.download_url orelse return error.InvalidHandle); }; + defer self.allocator.free(url); + var download: DataRequest = .{ .hf = self, .handle = handle, .uri = try .parse(url), .url = url }; + defer if (download.re_resolved) |owned| self.allocator.free(owned); + return range_read.performRangeRead(&self.governor, .{ + .request = .{ + // The file, not its signed URL: that is a credential, and + // a re-resolve replaces it. + .target = handle.uri, + .key = request.authorityOf(download.uri), + // A signed CDN URL can expire while the backend is held; + // the CDN then answers 401 or 403, which is not retryable in + // general but is here, once, after re-resolving it. + .retry_once = &.{ .unauthorized, .forbidden }, + }, + }, &download, DataRequest.prepare, data, offset, handle.size); + } - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, + const DataRequest = struct { + hf: *HF, + handle: *Handle, + /// Parsed from `url`, whose bytes it slices. + uri: std.Uri, + /// The read's own copy of the URL it sends. + url: []const u8, + /// The URL `reresolve` gave this read, owned by the read; at most + /// one per read. + re_resolved: ?[]const u8 = null, + + fn prepare(self: *DataRequest, attempt: request.Attempt, _: std.http.Header) anyerror!range_read.PreparedRequest { + if (attempt.previous_status) |status| switch (status) { + .unauthorized, .forbidden => { + if (self.re_resolved != null) return error.PermissionDenied; + try self.reresolve(); + }, + else => {}, + }; + return .{ + .uri = self.uri, + .authorization = self.hf.hubAuthorization(self.url), }; } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; - - return read_size; - } + /// Resolves the download URL again through the governed HEAD chain + /// and stores a copy on the handle, so later reads of the same file + /// use the fresh one too. Reads copy the handle's URL under + /// `hf.mutex`, so none still uses the one this replaces. + fn reresolve(self: *DataRequest) !void { + const allocator = self.hf.allocator; + const repo = try Repo.parse(self.handle.uri); + var url_buffer: [16 * 1024]u8 = undefined; + const resolved = try self.hf.resolveDownloadUrl(repo, &url_buffer); + // Everything that can fail comes before anything is handed over. + const owned = try allocator.dupe(u8, resolved); + errdefer allocator.free(owned); + const uri: std.Uri = try .parse(owned); + const for_handle = try allocator.dupe(u8, resolved); + { + self.hf.mutex.lockUncancelable(self.hf.base.inner); + defer self.hf.mutex.unlock(self.hf.base.inner); + if (self.handle.download_url) |previous| allocator.free(previous); + self.handle.download_url = for_handle; + } + self.re_resolved = owned; + self.url = owned; + self.uri = uri; + log.warn("hf: re-resolved the download URL of {s}", .{self.handle.uri}); + } + }; }; + +test "HF reports a path absent from the repository listing as FileNotFound" { + const allocator = std.testing.allocator; + var client: std.http.Client = .{ .allocator = allocator, .io = std.testing.io }; + defer client.deinit(); + var hf: HF = try .init(allocator, std.testing.io, &client, .{}); + defer hf.deinit(); + + // The listing `getOrFetchTree` would keep, so no request is sent. + var tree: std.ArrayList(TreeNode) = .empty; + try HF.insertTreeNode(allocator, &tree, .{ .oid = "", .path = "config.json", .type = "file", .size = 2 }); + try HF.insertTreeNode(allocator, &tree, .{ .oid = "", .path = "nested/model.safetensors", .type = "file", .size = 8 }); + try hf.trees.put(allocator, try allocator.dupe(u8, "owner/model@main"), tree); + + const io = hf.io(); + try std.testing.expectEqual(@as(u64, 2), (try std.Io.Dir.statFile(.cwd(), io, "owner/model/config.json", .{})).size); + try std.Io.Dir.access(.cwd(), io, "owner/model/nested/model.safetensors", .{}); + try std.testing.expectError(error.IsDir, std.Io.Dir.statFile(.cwd(), io, "owner/model/nested", .{})); + try std.testing.expectError(error.IsDir, std.Io.Dir.statFile(.cwd(), io, "owner/model", .{})); + // Opening a directory as a file sends no request. + try std.testing.expectError(error.IsDir, std.Io.Dir.openFile(.cwd(), io, "owner/model/nested", .{})); + try std.testing.expectError(error.IsDir, std.Io.Dir.openFile(.cwd(), io, "owner/model", .{})); + + try std.testing.expectError(error.FileNotFound, std.Io.Dir.statFile(.cwd(), io, "owner/model/missing.json", .{})); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.access(.cwd(), io, "owner/model/missing.json", .{})); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.openFile(.cwd(), io, "owner/model/missing.json", .{})); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.openDir(.cwd(), io, "owner/model/missing", .{})); +} + +test "HF sends its token to the Hub only" { + const allocator = std.testing.allocator; + var client: std.http.Client = .{ .allocator = allocator, .io = std.testing.io }; + defer client.deinit(); + var hf: HF = try .init(allocator, std.testing.io, &client, .{ .token = "hf_token" }); + defer hf.deinit(); + + try std.testing.expectEqualStrings("Bearer hf_token", hf.hubAuthorization("https://huggingface.co/owner/model/resolve/main/model.safetensors").override); + // Not the Hub: the CDNs its resolve URLs redirect to, a look-alike + // host, plain http. + for ([_][]const u8{ + "https://cas-bridge.xethub.hf.co/xet-bridge-us/0123?X-Amz-Signature=abc", + "https://cdn-lfs.huggingface.co/repos/00/11/0123", + "https://huggingface.co.evil.com/owner/model/resolve/main/model.safetensors", + "http://huggingface.co/owner/model/resolve/main/model.safetensors", + }) |url| try std.testing.expect(hf.hubAuthorization(url) == .omit); +} From 0ffe83052f8009bb87d9a0768997c15cec1d3374 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 4 Nov 2026 02:33:12 +0100 Subject: [PATCH 102/103] vfs: delete parallel_read and document the remote error contract HTTP, HF, S3 and GCS now all run their requests through request.zig, so vfs/parallel_read.zig has no user left and goes with its BUILD entry. A remote read's width is its caller's: a backend-local pool would multiply it and defeat the reader's memory bounds. vfs.zig's header named only HTTP as following the open and read error contract; it now states it for all four backends: a missing object is FileNotFound (404, 410, or a path absent from an HF repository's listing), a denied one PermissionDenied (401, 403), and S3 answers a principal without s3:ListBucket 403 for a missing key too, so a probe of an optional object treats both as absent. The header also says that a Loader may keep its source files open until its deinit, so the VFS and its backends must outlive it, and that it is the loader that asks fromIo. Remotes' doc points at the contract, and S3's HEAD doc now refers to the header instead of restating the 403 rule. No behaviour change; //vfs:test, //zml:test, //stdx:test and the examples build on CPU. Co-Authored-By: Claude Opus 5.5 --- vfs/BUILD.bazel | 1 - vfs/parallel_read.zig | 211 ------------------------------------------ vfs/s3.zig | 5 +- vfs/vfs.zig | 44 +++++---- 4 files changed, 26 insertions(+), 235 deletions(-) delete mode 100644 vfs/parallel_read.zig diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index 366c6488..352ce826 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -10,7 +10,6 @@ zig_library( "http.zig", "http_acceptance_test.zig", "mock_server.zig", - "parallel_read.zig", "range_read.zig", "request.zig", "s3.zig", diff --git a/vfs/parallel_read.zig b/vfs/parallel_read.zig deleted file mode 100644 index 70c9d89d..00000000 --- a/vfs/parallel_read.zig +++ /dev/null @@ -1,211 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -pub const InitOpts = struct { - chunk_size: usize, - num_workers: usize, - queue_capacity: usize, - max_retries: usize, - retry_initial_delay: std.Io.Duration, - retry_max_delay: std.Io.Duration, -}; - -pub const Status = union(enum) { - success, - retry_after: ?std.Io.Duration, - - pub fn retry() Status { - return .{ .retry_after = null }; - } -}; - -pub const BatchState = struct { - pending: std.atomic.Value(u32), - err: std.atomic.Value(u16) = .init(0), - - pub fn failed(batch: *BatchState) bool { - return batch.err.load(.acquire) != 0; - } - - pub fn fail(batch: *BatchState, err: anyerror) void { - _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); - } - - pub fn anyError(batch: *BatchState) ?anyerror { - const err = batch.err.load(.acquire); - return if (err == 0) null else @errorFromInt(err); - } - - pub fn completeOne(batch: *BatchState, io: std.Io) void { - const previous = batch.pending.fetchSub(1, .release); - std.debug.assert(previous > 0); - if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); - } - - pub fn waitUncancelable(batch: *BatchState, io: std.Io) void { - while (true) { - const pending = batch.pending.load(.acquire); - if (pending == 0) return; - io.futexWaitUncancelable(u32, &batch.pending.raw, pending); - } - } -}; - -pub fn Pool(comptime Job: type) type { - return struct { - const Self = @This(); - - group: std.Io.Group = .init, - job_queue: std.Io.Queue(Job), - queue_buf: []Job, - client: *std.http.Client, - prng: std.Random.DefaultPrng, - chunk_size: usize, - max_retries: usize, - retry_initial_delay: std.Io.Duration, - retry_max_delay: std.Io.Duration, - - pub fn init(self: *Self, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { - stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); - stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); - stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); - stdx.debug.assert(opts.retry_initial_delay.nanoseconds >= 0, "Pool retry initial delay must not be negative", .{}); - stdx.debug.assert(opts.retry_max_delay.nanoseconds >= 0, "Pool retry max delay must not be negative", .{}); - stdx.debug.assert(opts.retry_max_delay.nanoseconds >= opts.retry_initial_delay.nanoseconds, "Pool retry max delay must be at least the initial delay", .{}); - - const queue_buf = try allocator.alloc(Job, opts.queue_capacity); - errdefer allocator.free(queue_buf); - - self.* = .{ - .client = client, - .chunk_size = opts.chunk_size, - .max_retries = opts.max_retries, - .retry_initial_delay = opts.retry_initial_delay, - .retry_max_delay = opts.retry_max_delay, - .prng = .init(blk: { - var seed: u64 = undefined; - io.random(@ptrCast(&seed)); - break :blk seed; - }), - .job_queue = .init(queue_buf), - .queue_buf = queue_buf, - }; - errdefer self.group.cancel(io); - - for (0..opts.num_workers) |_| { - try self.group.concurrent(io, worker, .{ self, io }); - } - } - - pub fn deinit(self: *Self, allocator: std.mem.Allocator, io: std.Io) void { - self.job_queue.close(io); - self.group.cancel(io); - - allocator.free(self.queue_buf); - } - - fn worker(pool: *Self, io: std.Io) std.Io.Cancelable!void { - while (true) { - const job = pool.job_queue.getOne(io) catch break; - - var attempt: usize = 0; - while (true) { - if (job.batch.state.failed()) break; - const status = job.perform(pool.client) catch |err| { - job.batch.state.fail(err); - break; - }; - - switch (status) { - .success => break, - .retry_after => |duration| { - if (attempt >= pool.max_retries) { - job.batch.state.fail(error.RetriesExhausted); - break; - } - - io.sleep(duration orelse pool.backoffDuration(attempt), .awake) catch { - job.batch.state.fail(error.RetriesExhausted); - break; - }; - - attempt += 1; - }, - } - continue; - } - - job.batch.state.completeOne(io); - } - } - - fn backoffDuration(pool: *Self, attempt: usize) std.Io.Duration { - // FullJitter algorithm - const max_delay_ns: i96 = @min( - pool.retry_max_delay.toNanoseconds(), - pool.retry_initial_delay.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), - ); - const delay_ns = pool.prng.random().intRangeAtMost(i96, 0, max_delay_ns); - return .fromNanoseconds(delay_ns); - } - }; -} - -pub const ContentRange = struct { - start: u64, - end: u64, - total: u64, -}; - -pub fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; -} - -pub fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { - if (offset >= file_size) return 0; - - var requested: usize = 0; - for (data) |buf| requested += buf.len; - return @intCast(@min(file_size - offset, requested)); -} - -pub fn readChunk( - reader: *std.Io.Reader, - content_range: ?ContentRange, - file_offset: u64, - data: []const []u8, - chunk_offset: usize, - chunk_len: usize, -) !void { - if (content_range) |cr| { - if (cr.start > file_offset) return error.InvalidContentRange; - if (cr.start < file_offset) { - try reader.discardAll(file_offset - cr.start); - } - } - - var remaining = chunk_len; - var skip = chunk_offset; - - for (data) |buf| { - if (skip >= buf.len) { - skip -= buf.len; - continue; - } - - const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; - try reader.readSliceAll(destination); - remaining -= destination.len; - if (remaining == 0) break; - skip = 0; - } -} diff --git a/vfs/s3.zig b/vfs/s3.zig index 75a2d00c..65132a64 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -741,9 +741,8 @@ pub const S3 = struct { return request.perform(u64, &self.governor, .{ .target = url, .key = request.authorityOf(uri) }, &head, SizeRequest.attempt); } - /// The object's size: one signed HEAD (`ExchangeOptions.open`). S3 - /// answers a principal without `s3:ListBucket` 403 for a missing key too, - /// so a missing key may open as `PermissionDenied`. + /// The object's size: one signed HEAD (`ExchangeOptions.open`; S3's 403 + /// for a missing key: vfs.zig's header). const SizeRequest = struct { s3: *S3, uri: std.Uri, diff --git a/vfs/vfs.zig b/vfs/vfs.zig index 91011e23..8a52e763 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -2,28 +2,31 @@ //! `file://` URI is served by the inner io; a URI such as //! `hf://org/model/model.safetensors` or `s3://bucket/key` by the backend //! registered for its scheme (`registerBackend`, or the four of `Remotes`). -//! Readers take `io()` and need not know which they read; one that must (for -//! read sizes and direct I/O) asks `fromIo`. +//! Readers take `io()` and need not know which they read; one that must +//! (the loader, for read sizes and direct I/O) asks `fromIo`. //! //! The VFS, and every backend registered on it, must outlive each file -//! opened through it. +//! opened through it and each reader that keeps one open: a `Loader` may +//! keep its source files open until its `deinit`. //! -//! Errors of a remote path on a backend that runs its requests through -//! `request.zig` (`HTTP`, which serves `https://`): opening, stat-ing or -//! probing it (`access`) fails with `FileNotFound` for a missing object -//! (404, 410), `PermissionDenied` for a denied one (401, 403), `Canceled` -//! for a canceled task, and `SystemResources` when a request cannot -//! allocate. Any other failure of a request is logged at err with the -//! backend and the path, and surfaces as `Unexpected`. A positional read -//! that still fails after its retries is logged at err and surfaces as -//! `Unexpected`; a streaming read as `InputOutput`. A throttled backend -//! holds its requests rather than failing them, until its throttle budget -//! runs out (`request.Retry`). A remote path is read-only: creating or -//! deleting a file or a directory there, or opening a file for writing, is -//! logged at err and fails with `ReadOnlyFileSystem`. A local file fails as -//! the inner io does (a denied one is `AccessDenied` there); it may be read -//! past the page cache (`useDirectIo`). A backend on `request.zig` logs its -//! read totals at debug from its `deinit`. +//! Errors of a remote path, alike on `HTTP`, `HF`, `S3` and `GCS`: opening, +//! stat-ing or probing it (`access`) fails with `FileNotFound` for a missing +//! object (404, 410, or a path absent from an `HF` repository's listing), +//! `PermissionDenied` for a denied one (401, 403), `Canceled` for a canceled +//! task, and `SystemResources` when a request cannot allocate. Any other +//! failure of a request is logged at err with the backend and the path, and +//! surfaces as `Unexpected`. On S3 a principal without `s3:ListBucket` gets +//! 403 for a missing key too, so a probe of an optional object treats both +//! as absent. A positional read that still fails after its retries is +//! logged at err and surfaces as `Unexpected`; a streaming read as +//! `InputOutput`. A throttled backend holds its requests rather than failing +//! them, until its throttle budget runs out (`request.Retry`). A remote path +//! is read-only: creating or deleting a file or a directory there, or +//! opening a file for writing, is logged at err and fails with +//! `ReadOnlyFileSystem`. A local file fails as the inner io does (a denied +//! one is `AccessDenied` there); it may be read past the page cache +//! (`useDirectIo`). Each remote backend logs its read totals at debug from +//! its `deinit`. const std = @import("std"); const stdx = @import("stdx"); @@ -122,7 +125,8 @@ pub fn registerBackend(self: *VFS, scheme: []const u8, backend: Backend) std.mem /// `HTTP.init` for `https`), the client honoring the proxy variables. /// Initialized in place, since the backends point at the client; it must not /// move afterwards. Close every file opened through these schemes before -/// `deinit`; the VFS must not reach them after it. +/// `deinit` (deinitialize a `Loader` first); the VFS must not reach them +/// after it. Their errors follow this module's contract. pub const Remotes = struct { /// The proxies' allocations, which must outlive the client. arena: std.heap.ArenaAllocator, From 2accfb151507043d010e70286e6da928dd7c12a9 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 4 Nov 2026 01:34:41 +0100 Subject: [PATCH 103/103] docs: add the loader guide and a loading-weights tutorial step docs/learn/loader.md is the guide to zml.io.Loader: the recipe over a VFS with Remotes, placing tensors on a mesh or in host memory, composite tensors built from Source's shape ops, programs bound with bindProgram, and what each handle borrows and how a load fails. The errors get a short summary that points at Loader's doc comment, which holds the contract. The docs index, the README's taste of ZML, Buffer's doc and the headers of zml/io.zig and zml/io/loader.zig link it. write_first_model gains a last step, 'Loading weights': a few lines of Python write the layer's weight and bias to a .safetensors file, the layer binds them through a TensorStore view, and main loads them over a VFS with Remotes through the loader before calling the executable. The TensorStore paragraph now says the store holds the shapes and the loader fills the buffers. zml.io.next goes: nothing in zml or llmd names it any more. The tutorial's program and every snippet of the guide were built and run on 4 CPU devices from a scratch target that is not committed: the tutorial prints {6,11,16} from the Python-written checkpoint, and the placement (host_mapped included) and composite snippets load the values they describe. The program snippet stands alone: it defines its sources and its forward dequantizes by block and places its result as the output is, without which the load fails with InvalidBinding; on an 8x256 weight with one scale per row and 128 columns it loads every value as expected, row-sharded. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 + docs/README.md | 1 + docs/learn/loader.md | 210 ++++++++++++++++++++++++++++ docs/tutorials/write_first_model.md | 171 +++++++++++++++++++++- zml/buffer.zig | 2 +- zml/io.zig | 8 +- zml/io/loader.zig | 3 +- 7 files changed, 384 insertions(+), 13 deletions(-) create mode 100644 docs/learn/loader.md diff --git a/README.md b/README.md index 34a794a9..216fc165 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,8 @@ pub fn main(init: std.process.Init) !void { ``` The first `start` waits for whatever calibration is left. +[Loading checkpoints](./docs/learn/loader.md) gives the full recipe, with +remote checkpoints, placement, lifetimes and errors. For a full walkthrough, see: diff --git a/docs/README.md b/docs/README.md index 03369120..3a356509 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,6 +24,7 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## Learn more... - [ZML Concepts](./learn/concepts.md) : **Tensors, Models, Executables, etc. explained** +- [Loading checkpoints](./learn/loader.md) : **The loading recipe, placement, programs, lifetimes and errors** ## Contribute diff --git a/docs/learn/loader.md b/docs/learn/loader.md new file mode 100644 index 00000000..d97b363c --- /dev/null +++ b/docs/learn/loader.md @@ -0,0 +1,210 @@ +# Loading checkpoints + +`zml.io.Loader` reads a model's weights from its checkpoint into device +buffers. Model code binds each tensor to checkpoint bytes through a +`TensorStore` view; the loader fills the buffers, assembles the tensors made +of several checkpoint tensors, and runs the programs that compute others. +The doc comments of `zml/io/loader.zig`, `TensorStore.zig`, `Source.zig` and +`vfs/vfs.zig` are the contract; the internals are in the `//!` header of +each file `zml/io.zig` lists. + +## The recipe + +```zig +// Local paths, and hf://, s3://, gs:// and https:// through `Remotes`, +// configured from the environment (tokens, credentials, proxies). +var vfs: zml.io.VFS = try .init(allocator, init.io); +defer vfs.deinit(); +var remotes: zml.io.VFS.Remotes = undefined; // initialized in place +try remotes.init(allocator, &vfs, init.environ_map); +defer remotes.deinit(); +const io = vfs.io(); // for the platform, the registry and the loader + +const platform: *zml.Platform = try .auto(allocator, io, .{}); +defer platform.deinit(allocator, io); + +// Returns at once: calibration runs while the model is built and compiled. +var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); +defer loader.deinit(); + +// The checkpoint's headers. +var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); +defer registry.deinit(); +var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); +defer store.deinit(); + +const model: Model = .init(store.view()); // binds every tensor +// ... compile the model ... + +var buffers = try zml.mem.bufferize(allocator, &model); +defer zml.mem.deinitBufferized(allocator, &buffers); +try loader.start(&model, &buffers, &store, .none); +try loader.wait(); // logs "Loaded weights [bytes, time, rate]" +``` + +`.checkpoint` lets the VFS backend serving the path size the reads, and the +loader place its pinned memory for how a local checkpoint reads; without it, +or without a VFS, reads are sized for local files. `start` adds its own items +to the progress node it is given. Several loads may be started before one +`wait`, from one store or several. A second checkpoint, such as a drafter's, +joins the registry under a prefix before the store takes it: +`registry.addRepo(io, drafter_repo, "drafter")`, then +`store.view().withPrefix("drafter")` reaches its tensors. + +## Placement + +A tensor is placed by its shape. `View.createTensor(subkey, tags, .tp, +partitioning)` places the checkpoint tensor on the mesh named `.tp` among the +store's `meshes` (`View.mesh(.tp)` returns it), each tag of `partitioning` +naming the mesh axis that splits it; `createReplicatedTensor` puts it whole +on every device. It lands in default device memory, unless `View.setMemory`, +called after binding, names a host kind (`TensorStore.Memory`): +`.host_pinned` gives buffers in pinned host memory, and `.host_mapped` gives +device buffers that are views of pinned host memory, which kernels read in +place. The second suits a table too large for device memory, of which each +step reads a few rows; the memory goes with the last view. XLA counts those +views against device memory: give their size on one device, +`view.hostMappedBytes()`, back through the compile option +`extra_device_memory_bytes`. + +```zig +const tp = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); +var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tp} }); +const view = store.view(); +const w = view.createTensor("w", .{ .out, .in }, .tp, .{ .out = .model }); +const b = view.createReplicatedTensor("b", .{.out}); +const rows = view.createReplicatedTensor("rows", .{ .row, .d }); +view.setMemory(rows, .host_mapped); +``` + +`View.get(subkey)` returns the registry's entry for a key, or null, and +`count()` the number of keys under the view's prefix. Binding cannot fail: +`createTensor` and `source` panic on a missing key, `maybeCreateTensor` and +`maybeSource` return null. `TensorStore.Options.quantization` +(`Quantization.Target.of(platform)`) is required to bind block-FP8 weights. + +## Composite tensors + +A composite tensor is one destination filled from several checkpoint byte +ranges, every byte no range covers set to zero, with no program and no +buffer per source. It is a `TensorStore.Source`: a checkpoint tensor +(`View.source(subkey, tags)`) and `zml.Tensor`'s shape ops over it, with the +same names, tags and arguments: `slice`, `split`, `chunkExact`, `splitAxis`, +`merge`, `reshape`, `transpose`, `pad` (with zeros), `broad`, `bitCast`, +`concatenate` and `stack`. Placement comes last, after every op: +`withPartitioning(mesh, partitioning)`, keyed by the source's final tags, or +`as(shape)`, a shape built elsewhere with the source's element count and +dtype. Both return a `Source.Placed`, which has no shape ops (its `.shape` +is the placed shape) and drops its placement with `unplaced()`; its +`bind()` makes the model tensor. + +```zig +const Source = zml.io.TensorStore.Source; +// a, b and c end to end along the rows, padded to 16 rows, row-sharded. +const abc = Source.concatenate(&.{ view.source("a", .{ .row, .d }), view.source("b", .{ .row, .d }), view.source("c", .{ .row, .d }) }, .row); +const table = abc.pad(0, .{ .row = Source.Pad{ .high = 16 - abc.dim(.row) } }) + .withPartitioning(.tp, .{ .row = .model }).bind(); +// q, k and v cut in one group per device and interleaved, so that each +// device holds its rows of every projection. +var members: [3]Source = undefined; +for (&members, [_][]const u8{ "q.weight", "k.weight", "v.weight" }) |*m, key| { + m.* = view.source(key, .{ .dout, .d }).splitAxis(.dout, .{ .group = devices, .r = -1 }); +} +const qkv = Source.concatenate(&members, .r).merge(.{ .dout = .{ .group, .r } }).withPartitioning(.tp, .{ .dout = .model }).bind(); +``` + +Sources live in their store's arena, and sources of two stores do not mix. +The loader only moves bytes: large runs are cheap, runs under 4 KiB cost +host copies. For a layout of many small runs, a program over sources read +in large runs may load faster, at the cost of a compile and staging memory +(measured in the header of `zml/io/direct_loader.zig`). + +## Programs + +What computes rather than moves bytes is a program: a dtype conversion, a +dequantization, a global scale. A program is a value of its own type, plain +data (checked at compile time), with a `forward` method; its type name +names the compiled module in traces and logs. + +```zig +const Dequantize = struct { + block_size: i64, + + pub fn forward(self: Dequantize, output: zml.Shape, inputs: []const zml.Tensor) zml.Tensor { + // One scale per row and block of `block_size` columns. + const blocks = inputs[0].splitAxis(.in, .{ .block = -1, .k = self.block_size }); + const y = blocks.mul(inputs[1].broad(blocks.shape())).merge(.{ .in = .{ .block, .k } }); + return y.convert(output.dtype()).withPartitioning(output._sharding.mesh.?, output._sharding.partition); + } +}; +const w = view.source("w", .{ .out, .in }); +const s = view.source("w_scale", .{ .out, .block }); +// Each device reads its rows of `w`, and all of `s`. +const placed = w.shape().withPartitioning(tp, .{ .out = .model }); +const weight = Source.bindProgram(&.{ w.as(placed), s.withPartitioning(.replicated, .{}) }, placed.withDtype(.bf16), Dequantize{ .block_size = 128 }); +``` + +`bindProgram(inputs, output, program)` binds a tensor of `output` that +`program.forward(output, inputs)` computes during the load and returns +placed as `output` is. Each input is a `Source.Placed` and lands placed by +its own shape, so every device reads only its slice; a whole checkpoint +tensor, replicated, is +`view.source(key, null).withPartitioning(.replicated, .{})`. `start` +starts one compile for all the tensors of an equal program (same type, +equal value) over the same placed shapes. `wait` lands the inputs in groups +of at most `Loader.Options.program_group_bytes` of outputs (8 GiB), runs +the programs on the calling task and frees the inputs. While other groups +are pending, a group waits until the devices have room for it; once none +is pending, it goes anyway, and a group that does not fit a device fails +`wait` with the device's error. Before `wait` returns, the load gives the +device memory it freed, inputs and temporaries, back to the devices; XLA's +VMM allocator, the default on CUDA and ROCm, would otherwise keep counting +it as in use. A program's output lands in default device memory: +`setMemory` panics on it. + +## Lifetime and failure + +| Handle | Borrows | Until | +| --- | --- | --- | +| `Loader` | the platform; `io`, and the VFS behind it, through which it may keep its source files open | its `deinit` | +| `Loader.start` | the store, the buffers and the progress node | `wait` or `cancel` returns | +| `TensorStore` | the meshes; it owns the registry's entries, the bindings, sources and programs | its `deinit` | +| `View`, `Source`, `Source.Placed` | their store, which must not move | their last use | +| `VFS.Remotes` | the VFS; it must not move | its `deinit`, once every file opened through it is closed | +| Loaded buffers | the platform | `deinitBufferized` | + +`init` reads its options and `start` the model during the call only. The +loader copies each file's path, so nothing of the store is kept past `wait`. +The loaded buffers are the caller's and outlive the loader, which holds its +pinned memory and workers until `deinit`: the recipe's deferred `deinit` +keeps them until the function returns, which suits a program that does +little after the load; one that keeps running frees the loader right after +`wait`, or loads in a block of its own. Deferred cleanup in the recipe's +order is right; if code between `start` and `wait` can fail, +`errdefer loader.cancel();` after `start`. The loader is a value. Start and +wait on one task: the backend reads and transfers concurrently. + +Every error of `start` or `wait` fails the loader: everything started is +canceled, and every later call returns the first error. `cancel` gives up on +what is started without running its programs. +Whatever failed, each buffer is empty or holds shards whose contents are +unspecified: read none, and free the tree with `zml.mem.deinitBufferized`. +That holds for the buffers `bufferize` made, which `start` empties again. + +## Errors + +`Loader.init` returns `InvalidOptions`, logged with the option it refuses, +or `OutOfMemory` when it cannot copy a local checkpoint's path; an error of +the backend's set-up is returned by the first `start`. +`start` and `wait` return the error where it was detected, under its own +name, and that is where it is logged, once, at err, with what explains it; +a plain allocation failure and `Canceled` are returned unlogged. + +A model whose bindings are wrong is `InvalidBinding`; the log names the +tensor (for a program's output, the program), the key its source reads and +the rule it breaks (`Loader.start`, `Loader.wait`). + +The other errors a caller can act on are listed in `Loader`'s doc comment: +a missing or denied file, a file that ends inside a tensor, host or device +memory that runs out, an `io` without concurrency, and `Canceled`. PJRT, +compile and remote failures pass through as their source names them. diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 69b54590..fc394979 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -6,7 +6,8 @@ - clone ZML to work directly within the prepared example folder - add Zig code to implement our model - add some Bazel to integrate our code with ZML -- no weights files or anything external is required for this example +- no weights files or anything external is required, until the last step + loads the weights from a checkpoint The reason we're doing our exercise in the `examples` folder is because it's especially prepared for new ZML projects. It contains everything needed for ZML @@ -58,10 +59,12 @@ _Module_. For compilation, only the _Tensors_ are required. No actual tensor dat is needed at this step. This is important for large models: we can compile them while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ loads everything -that is required to make _Tensors_ and later materialize them to _Buffers_. In our -example though, we won't even use a _TensorStore_ as it's optional. We will directly -create tensors using their shape and data type. +To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ holds the shapes +of a checkpoint's tensors, so that we can make _Tensors_, and the _Loader_ later +materializes them to _Buffers_. At first we won't use them: we will directly +create tensors using their shape and data type, and fill their buffers from +slices. The last step, [Loading weights](#loading-weights), reads them from a +checkpoint instead. After compilation is done, we get what we call an _Executable_. From this _Executable_ we can create _Args_ and _Results_, two structs that will store respectively the inputs @@ -411,10 +414,168 @@ pub fn main(init: std.process.Init) !void { } ``` +## Loading weights + +Real models read their weights from a checkpoint. Let's write one for our +layer, a `.safetensors` file holding `weight` and `bias`, with a few lines of +Python: + +```python +import json, struct + +weight = struct.pack("<3e", 1.0, 2.0, 3.0) # three f16 values +bias = struct.pack("<3e", 1.0, 1.0, 1.0) +header = json.dumps({ + "weight": {"dtype": "F16", "shape": [3], "data_offsets": [0, 6]}, + "bias": {"dtype": "F16", "shape": [3], "data_offsets": [6, 12]}, +}).encode() +with open("/tmp/layer.safetensors", "wb") as f: + f.write(struct.pack("