diff --git a/README.md b/README.md index d9e68b81..216fc165 100644 --- a/README.md +++ b/README.md @@ -162,29 +162,6 @@ const Mnist = struct { }; } - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - meshes: []const *const zml.Mesh, - ) !zml.Bufferized(Mnist) { - return zml.io.load(Mnist, self, allocator, io, platform, store, .{ - .meshes = meshes, - .parallelism = 1, - .dma_chunks = 1, - .dma_chunk_size = 16 * 1024 * 1024, - }); - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { - self.fc1.weight.deinit(); - self.fc1.bias.deinit(); - self.fc2.weight.deinit(); - self.fc2.bias.deinit(); - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); @@ -195,8 +172,46 @@ const Mnist = struct { return x.argMax(0).indices.convert(.u8); } }; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const path = ...; + + // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + const io = vfs.io(); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + // Returns at once: the loader calibrates while the model is compiled. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); + defer loader.deinit(); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); + defer store.deinit(); + const model: Mnist = .init(store.view()); + + const input: zml.Tensor = .init(.{ 28, 28 }, .u8); + var exe = try platform.compile(allocator, io, model, .forward, .{input}, .{}); + defer exe.deinit(); + + var weights = try zml.mem.bufferize(allocator, &model); + defer zml.mem.deinitBufferized(allocator, &weights); + try loader.start(&model, &weights, &store, .none); + try loader.wait(); // logs "Loaded weights" + + // ... call `exe` on the weights and an image +} ``` +The first `start` waits for whatever calibration is left. +[Loading checkpoints](./docs/learn/loader.md) gives the full recipe, with +remote checkpoints, placement, lifetimes and errors. + For a full walkthrough, see: - [Getting Started](./docs/tutorials/getting_started.md) diff --git a/docs/README.md b/docs/README.md index 03369120..3a356509 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,6 +24,7 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## Learn more... - [ZML Concepts](./learn/concepts.md) : **Tensors, Models, Executables, etc. explained** +- [Loading checkpoints](./learn/loader.md) : **The loading recipe, placement, programs, lifetimes and errors** ## Contribute diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 3d246f11..d2c63bc1 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -160,7 +160,6 @@ bazel build //my_project:torch2zml ./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Llama-3.2-1B-Instruct.activations.safetensors info: Found 564 activations in /Users/corendos/dev/zml/python/temp/Llama-3.2-1B-Instruct.activations.safetensors -debug(zml/safetensors): Added new metadata key=format with value=pt info: Found 146 activations in /Users/corendos/models/Llama-3.2-1B-Instruct/model.safetensors ``` @@ -190,7 +189,7 @@ convert it to a `TensorStore`: ```zig pub fn main(init: std.process.Init) !void { ... - var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); + var model_store: zml.io.TensorStore = .init(allocator, &model_registry, .{}); defer model_store.deinit(); } ``` @@ -206,17 +205,17 @@ pub fn main(init: std.process.Init) !void { const mlp: Mlp = .{ .up_proj = .init( mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), + mlp_view.maybeCreateTensor("up_proj.bias", .{.dout}, .replicated, .{}), .d, ), .gate_proj = .init( mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), + mlp_view.maybeCreateTensor("gate_proj.bias", .{.dout}, .replicated, .{}), .d, ), .down_proj = .init( mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), - mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), + mlp_view.maybeCreateTensor("down_proj.bias", .{.d}, .replicated, .{}), .dout, ), }; @@ -235,15 +234,16 @@ Python script. Alternatively, Huggingface's web-interface allows to peek into ## Loading an individual layer Currently, the layer is initialized but the weights are still on the disk. -ZML provides a convenience to automatically load weights from an initialized model: +ZML provides a convenience to load weights from an initialized model in a test: ```zig pub fn main(init: std.process.Init) !void { ... var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); - var mlp_weights = try zml.io.load(Mlp, &mlp, allocator, io, platform, &model_store, .auto); + defer platform.deinit(allocator, io); + var mlp_weights = try zml.testing.load(allocator, io, platform, &mlp, &model_store); + defer zml.mem.deinitBufferized(allocator, &mlp_weights); } ``` @@ -287,7 +287,7 @@ pub const Wrapper = struct { pub fn main(init: std.process.Init) !void { ... - var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); + var activations_store: zml.io.TensorStore = .init(allocator, &activations_registry, .{}); defer activations_store.deinit(); try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index cc11ffbb..9f18b4a7 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -17,7 +17,7 @@ access token to log into your account on Huggingface, directly from `bazel`, in * the token looks something like `hf_abCdEfGhijKlM`. * store the token on your machine (replace the placeholder with your actual token): -You can use the `HUGGINGFACE_TOKEN` environment variable to store the token or use +You can use the `HF_TOKEN` environment variable to store the token or use its standard location: ``` mkdir -p $HOME/.cache/huggingface/; echo > "$HOME/.cache/huggingface/token" @@ -29,6 +29,6 @@ Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! ``` # requires token in $HOME/.cache/huggingface/token, as created by the -# `hf auth login` command, or the `HUGGINGFACE_TOKEN` environment variable. +# `hf auth login` command, or the `HF_TOKEN` environment variable. bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` diff --git a/docs/learn/loader.md b/docs/learn/loader.md new file mode 100644 index 00000000..d97b363c --- /dev/null +++ b/docs/learn/loader.md @@ -0,0 +1,210 @@ +# Loading checkpoints + +`zml.io.Loader` reads a model's weights from its checkpoint into device +buffers. Model code binds each tensor to checkpoint bytes through a +`TensorStore` view; the loader fills the buffers, assembles the tensors made +of several checkpoint tensors, and runs the programs that compute others. +The doc comments of `zml/io/loader.zig`, `TensorStore.zig`, `Source.zig` and +`vfs/vfs.zig` are the contract; the internals are in the `//!` header of +each file `zml/io.zig` lists. + +## The recipe + +```zig +// Local paths, and hf://, s3://, gs:// and https:// through `Remotes`, +// configured from the environment (tokens, credentials, proxies). +var vfs: zml.io.VFS = try .init(allocator, init.io); +defer vfs.deinit(); +var remotes: zml.io.VFS.Remotes = undefined; // initialized in place +try remotes.init(allocator, &vfs, init.environ_map); +defer remotes.deinit(); +const io = vfs.io(); // for the platform, the registry and the loader + +const platform: *zml.Platform = try .auto(allocator, io, .{}); +defer platform.deinit(allocator, io); + +// Returns at once: calibration runs while the model is built and compiled. +var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); +defer loader.deinit(); + +// The checkpoint's headers. +var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); +defer registry.deinit(); +var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); +defer store.deinit(); + +const model: Model = .init(store.view()); // binds every tensor +// ... compile the model ... + +var buffers = try zml.mem.bufferize(allocator, &model); +defer zml.mem.deinitBufferized(allocator, &buffers); +try loader.start(&model, &buffers, &store, .none); +try loader.wait(); // logs "Loaded weights [bytes, time, rate]" +``` + +`.checkpoint` lets the VFS backend serving the path size the reads, and the +loader place its pinned memory for how a local checkpoint reads; without it, +or without a VFS, reads are sized for local files. `start` adds its own items +to the progress node it is given. Several loads may be started before one +`wait`, from one store or several. A second checkpoint, such as a drafter's, +joins the registry under a prefix before the store takes it: +`registry.addRepo(io, drafter_repo, "drafter")`, then +`store.view().withPrefix("drafter")` reaches its tensors. + +## Placement + +A tensor is placed by its shape. `View.createTensor(subkey, tags, .tp, +partitioning)` places the checkpoint tensor on the mesh named `.tp` among the +store's `meshes` (`View.mesh(.tp)` returns it), each tag of `partitioning` +naming the mesh axis that splits it; `createReplicatedTensor` puts it whole +on every device. It lands in default device memory, unless `View.setMemory`, +called after binding, names a host kind (`TensorStore.Memory`): +`.host_pinned` gives buffers in pinned host memory, and `.host_mapped` gives +device buffers that are views of pinned host memory, which kernels read in +place. The second suits a table too large for device memory, of which each +step reads a few rows; the memory goes with the last view. XLA counts those +views against device memory: give their size on one device, +`view.hostMappedBytes()`, back through the compile option +`extra_device_memory_bytes`. + +```zig +const tp = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); +var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tp} }); +const view = store.view(); +const w = view.createTensor("w", .{ .out, .in }, .tp, .{ .out = .model }); +const b = view.createReplicatedTensor("b", .{.out}); +const rows = view.createReplicatedTensor("rows", .{ .row, .d }); +view.setMemory(rows, .host_mapped); +``` + +`View.get(subkey)` returns the registry's entry for a key, or null, and +`count()` the number of keys under the view's prefix. Binding cannot fail: +`createTensor` and `source` panic on a missing key, `maybeCreateTensor` and +`maybeSource` return null. `TensorStore.Options.quantization` +(`Quantization.Target.of(platform)`) is required to bind block-FP8 weights. + +## Composite tensors + +A composite tensor is one destination filled from several checkpoint byte +ranges, every byte no range covers set to zero, with no program and no +buffer per source. It is a `TensorStore.Source`: a checkpoint tensor +(`View.source(subkey, tags)`) and `zml.Tensor`'s shape ops over it, with the +same names, tags and arguments: `slice`, `split`, `chunkExact`, `splitAxis`, +`merge`, `reshape`, `transpose`, `pad` (with zeros), `broad`, `bitCast`, +`concatenate` and `stack`. Placement comes last, after every op: +`withPartitioning(mesh, partitioning)`, keyed by the source's final tags, or +`as(shape)`, a shape built elsewhere with the source's element count and +dtype. Both return a `Source.Placed`, which has no shape ops (its `.shape` +is the placed shape) and drops its placement with `unplaced()`; its +`bind()` makes the model tensor. + +```zig +const Source = zml.io.TensorStore.Source; +// a, b and c end to end along the rows, padded to 16 rows, row-sharded. +const abc = Source.concatenate(&.{ view.source("a", .{ .row, .d }), view.source("b", .{ .row, .d }), view.source("c", .{ .row, .d }) }, .row); +const table = abc.pad(0, .{ .row = Source.Pad{ .high = 16 - abc.dim(.row) } }) + .withPartitioning(.tp, .{ .row = .model }).bind(); +// q, k and v cut in one group per device and interleaved, so that each +// device holds its rows of every projection. +var members: [3]Source = undefined; +for (&members, [_][]const u8{ "q.weight", "k.weight", "v.weight" }) |*m, key| { + m.* = view.source(key, .{ .dout, .d }).splitAxis(.dout, .{ .group = devices, .r = -1 }); +} +const qkv = Source.concatenate(&members, .r).merge(.{ .dout = .{ .group, .r } }).withPartitioning(.tp, .{ .dout = .model }).bind(); +``` + +Sources live in their store's arena, and sources of two stores do not mix. +The loader only moves bytes: large runs are cheap, runs under 4 KiB cost +host copies. For a layout of many small runs, a program over sources read +in large runs may load faster, at the cost of a compile and staging memory +(measured in the header of `zml/io/direct_loader.zig`). + +## Programs + +What computes rather than moves bytes is a program: a dtype conversion, a +dequantization, a global scale. A program is a value of its own type, plain +data (checked at compile time), with a `forward` method; its type name +names the compiled module in traces and logs. + +```zig +const Dequantize = struct { + block_size: i64, + + pub fn forward(self: Dequantize, output: zml.Shape, inputs: []const zml.Tensor) zml.Tensor { + // One scale per row and block of `block_size` columns. + const blocks = inputs[0].splitAxis(.in, .{ .block = -1, .k = self.block_size }); + const y = blocks.mul(inputs[1].broad(blocks.shape())).merge(.{ .in = .{ .block, .k } }); + return y.convert(output.dtype()).withPartitioning(output._sharding.mesh.?, output._sharding.partition); + } +}; +const w = view.source("w", .{ .out, .in }); +const s = view.source("w_scale", .{ .out, .block }); +// Each device reads its rows of `w`, and all of `s`. +const placed = w.shape().withPartitioning(tp, .{ .out = .model }); +const weight = Source.bindProgram(&.{ w.as(placed), s.withPartitioning(.replicated, .{}) }, placed.withDtype(.bf16), Dequantize{ .block_size = 128 }); +``` + +`bindProgram(inputs, output, program)` binds a tensor of `output` that +`program.forward(output, inputs)` computes during the load and returns +placed as `output` is. Each input is a `Source.Placed` and lands placed by +its own shape, so every device reads only its slice; a whole checkpoint +tensor, replicated, is +`view.source(key, null).withPartitioning(.replicated, .{})`. `start` +starts one compile for all the tensors of an equal program (same type, +equal value) over the same placed shapes. `wait` lands the inputs in groups +of at most `Loader.Options.program_group_bytes` of outputs (8 GiB), runs +the programs on the calling task and frees the inputs. While other groups +are pending, a group waits until the devices have room for it; once none +is pending, it goes anyway, and a group that does not fit a device fails +`wait` with the device's error. Before `wait` returns, the load gives the +device memory it freed, inputs and temporaries, back to the devices; XLA's +VMM allocator, the default on CUDA and ROCm, would otherwise keep counting +it as in use. A program's output lands in default device memory: +`setMemory` panics on it. + +## Lifetime and failure + +| Handle | Borrows | Until | +| --- | --- | --- | +| `Loader` | the platform; `io`, and the VFS behind it, through which it may keep its source files open | its `deinit` | +| `Loader.start` | the store, the buffers and the progress node | `wait` or `cancel` returns | +| `TensorStore` | the meshes; it owns the registry's entries, the bindings, sources and programs | its `deinit` | +| `View`, `Source`, `Source.Placed` | their store, which must not move | their last use | +| `VFS.Remotes` | the VFS; it must not move | its `deinit`, once every file opened through it is closed | +| Loaded buffers | the platform | `deinitBufferized` | + +`init` reads its options and `start` the model during the call only. The +loader copies each file's path, so nothing of the store is kept past `wait`. +The loaded buffers are the caller's and outlive the loader, which holds its +pinned memory and workers until `deinit`: the recipe's deferred `deinit` +keeps them until the function returns, which suits a program that does +little after the load; one that keeps running frees the loader right after +`wait`, or loads in a block of its own. Deferred cleanup in the recipe's +order is right; if code between `start` and `wait` can fail, +`errdefer loader.cancel();` after `start`. The loader is a value. Start and +wait on one task: the backend reads and transfers concurrently. + +Every error of `start` or `wait` fails the loader: everything started is +canceled, and every later call returns the first error. `cancel` gives up on +what is started without running its programs. +Whatever failed, each buffer is empty or holds shards whose contents are +unspecified: read none, and free the tree with `zml.mem.deinitBufferized`. +That holds for the buffers `bufferize` made, which `start` empties again. + +## Errors + +`Loader.init` returns `InvalidOptions`, logged with the option it refuses, +or `OutOfMemory` when it cannot copy a local checkpoint's path; an error of +the backend's set-up is returned by the first `start`. +`start` and `wait` return the error where it was detected, under its own +name, and that is where it is logged, once, at err, with what explains it; +a plain allocation failure and `Canceled` are returned unlogged. + +A model whose bindings are wrong is `InvalidBinding`; the log names the +tensor (for a program's output, the program), the key its source reads and +the rule it breaks (`Loader.start`, `Loader.wait`). + +The other errors a caller can act on are listed in `Loader`'s doc comment: +a missing or denied file, a file that ends inside a tensor, host or device +memory that runs out, an `io` without concurrency, and `Canceled`. PJRT, +compile and remote failures pass through as their source names them. diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 312c9afc..fc394979 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -6,7 +6,8 @@ - clone ZML to work directly within the prepared example folder - add Zig code to implement our model - add some Bazel to integrate our code with ZML -- no weights files or anything external is required for this example +- no weights files or anything external is required, until the last step + loads the weights from a checkpoint The reason we're doing our exercise in the `examples` folder is because it's especially prepared for new ZML projects. It contains everything needed for ZML @@ -58,10 +59,12 @@ _Module_. For compilation, only the _Tensors_ are required. No actual tensor dat is needed at this step. This is important for large models: we can compile them while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ loads everything -that is required to make _Tensors_ and later materialize them to _Buffers_. In our -example though, we won't even use a _TensorStore_ as it's optional. We will directly -create tensors using their shape and data type. +To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ holds the shapes +of a checkpoint's tensors, so that we can make _Tensors_, and the _Loader_ later +materializes them to _Buffers_. At first we won't use them: we will directly +create tensors using their shape and data type, and fill their buffers from +slices. The last step, [Loading weights](#loading-weights), reads them from a +checkpoint instead. After compilation is done, we get what we call an _Executable_. From this _Executable_ we can create _Args_ and _Results_, two structs that will store respectively the inputs @@ -175,9 +178,9 @@ var executable = try platform.compile(allocator, io, layer, .forward, .{input}, defer executable.deinit(); ``` -You might wonder what this `sharding` variable is for ? ZML supports sharding tensors across multiple devices, -and the `compile()` function needs to know how the tensors are sharded in order to compile the module correctly. -For this simple example, we just replicate the tensors across all devices for simplicity. +ZML supports sharding tensors across multiple devices, and `compile()` reads how +each tensor is sharded from its shape. A tensor created from a shape alone, as +here, is replicated on every device. ### Creating the "bufferized" Model @@ -191,8 +194,8 @@ provides ways to have extra-low loading times). const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 2.0, 3.0})); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 1.0, 1.0})); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); @@ -213,7 +216,7 @@ To create the _Args_ and _Results_, the _Executable_ exposes two conveniences. ```zig // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); -var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); +var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); // create the Args and Results structs @@ -227,7 +230,7 @@ defer results.deinit(allocator); args.set(.{ layer_buffers, input_buffer }); // call our executable -executable.call(io, args, &results, .{}); +try executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); @@ -372,15 +375,15 @@ pub fn main(init: std.process.Init) !void { const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 2.0, 3.0 })); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 1.0, 1.0 })); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); // create the Args and Results structs @@ -394,7 +397,7 @@ pub fn main(init: std.process.Init) !void { args.set(.{ layer_buffers, input_buffer }); // call our executable - executable.call(io, args, &results, .{}); + try executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); @@ -411,10 +414,168 @@ pub fn main(init: std.process.Init) !void { } ``` +## Loading weights + +Real models read their weights from a checkpoint. Let's write one for our +layer, a `.safetensors` file holding `weight` and `bias`, with a few lines of +Python: + +```python +import json, struct + +weight = struct.pack("<3e", 1.0, 2.0, 3.0) # three f16 values +bias = struct.pack("<3e", 1.0, 1.0, 1.0) +header = json.dumps({ + "weight": {"dtype": "F16", "shape": [3], "data_offsets": [0, 6]}, + "bias": {"dtype": "F16", "shape": [3], "data_offsets": [6, 12]}, +}).encode() +with open("/tmp/layer.safetensors", "wb") as f: + f.write(struct.pack(" [replicated|sharded|tp] +//! bazel run //examples/io:load_check -- dma-bench +//! +//! The checkpoint is a model directory, a .safetensors file or a +//! .safetensors.index.json, local or remote (hf://, s3://, gs://, https://, +//! with the credentials and proxies of the environment). Each tensor is +//! placed on the `tp` mesh, all the platform's devices on its `model` axis, +//! by the mode: +//! - `replicated`: whole on every device; +//! - `sharded`, the default: split along its first axis, which the devices +//! must divide, whatever the model (DeepSeek-V4.x cannot bind on 2 or 4 +//! devices); +//! - `tp`: the input axis of `o_proj` and `down_proj` weights, as tensor +//! parallelism shards them (column-sharded: every device's bytes are +//! strided in the checkpoint), the first axis of the others. +//! A scalar is replicated in every mode. +//! +//! `dma-bench` loads nothing: it runs the direct backend's DMA calibration +//! (`zml.io.dma_calibration.measure`) on the platform and logs the +//! block it selects. A platform that loads through the buffered backend has +//! no DMA to measure, and CPU selects the smallest block unmeasured. Its +//! knobs, beside the platform's below, are the screen's options, at their +//! defaults when unset: +//! - `ZML_DMA_BENCH_BLOCK_MIB=a,b,...`: the candidate blocks in MiB, each +//! at most 32 (2, 4, 8, 16 and 32); a load takes a single one (below); +//! - `ZML_DMA_BENCH_BLOCK_PARALLELISM=n`: transfers in flight per device, +//! at most 32 (8); +//! - `ZML_DMA_BENCH_WINDOW_MS=n`: a screen window's minimum duration (2); +//! - `ZML_DMA_BENCH_MIN_TRANSFERS=n`: the transfers a window waits for on +//! the measured device, 0 for none (32); +//! - `ZML_DMA_BENCH_BLOCK_TOLERANCE=f`: the smallest block within this +//! fraction of the peak rate is selected (0.08). +//! +//! The loader logs the bytes it loaded, its time and rate at info, and its +//! stats at debug, on the `zml/io/load` scope; `zml/io` logs the DMA +//! calibration's pick with its measured rate on one device (`dma_bench`); +//! `zml/vfs` logs the remote read totals. This tool enables all three. +//! +//! Knobs, from the environment: +//! - `ZML_LOAD_CHECK=n`: once loaded, read every n-th tensor and the largest +//! one back to host and compare them with the checkpoint's bytes, every +//! replica of a tensor each device holds whole. 1 checks every tensor; +//! unset or 0, none. A difference is `LoadContentMismatch`, logged. +//! - `ZML_LOAD_READ_PARALLELISM=n`: the loader's concurrent source reads +//! (`Loader.Options.read_parallelism`); unset, the checkpoint's default +//! width, 16 locally and 32 remotely. +//! - `ZML_DMA_BENCH_BLOCK_MIB=n`: the direct backend's DMA block, in MiB +//! (`Loader.Options.dma_block_size`); unset, calibration picks one. +//! - `ZML_DIRECT_IO=off|on|auto`: direct I/O for local checkpoint files +//! (`Loader.Options.direct_io`, `VFS.DirectIo`); `auto` when unset. +//! - `ZML_LOAD_MEMORY=default|host_pinned|host_mapped`: where every tensor +//! is delivered (`TensorStore.Memory`, `View.setMemory`); `default`, device +//! memory, when unset. The loader refuses a kind the platform cannot take +//! (`InvalidBinding`). +//! - `ZML_GPU_MEMORY_FRACTION=f`: a preallocated BFC pool of that fraction +//! of each device's memory on CUDA, ROCm and oneAPI, instead of the +//! platform's allocator defaults, for a run that observes the loader's +//! admission on a device with room. +//! - `ZML_CPU_DEVICE_COUNT=n`: the CPU platform's devices (4 by default). +const std = @import("std"); + +const zml = @import("zml"); + +const log = std.log.scoped(.load_check); + +pub const std_options: std.Options = .{ + .log_level = .info, + .log_scope_levels = &.{ + .{ .scope = .@"zml/io/load", .level = .debug }, + .{ .scope = .@"zml/io", .level = .debug }, + .{ .scope = .@"zml/vfs", .level = .debug }, + }, +}; + +const Mode = enum { replicated, sharded, tp }; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + var args = init.minimal.args.iterate(); + _ = args.next(); // skip program name + const path = args.next() orelse { + log.err("usage: load_check [replicated|sharded|tp], or load_check dma-bench", .{}); + return error.MissingPath; + }; + if (std.mem.eql(u8, path, "dma-bench")) return dmaBench(init); + const mode = std.meta.stringToEnum(Mode, args.next() orelse "sharded") orelse { + log.err("the mode must be replicated, sharded or tp", .{}); + return error.InvalidArgument; + }; + const check_stride = try envNumber(usize, init.environ_map, "ZML_LOAD_CHECK") orelse 0; + const direct_io = std.meta.stringToEnum(zml.io.VFS.DirectIo, init.environ_map.get("ZML_DIRECT_IO") orelse "auto") orelse { + log.err("ZML_DIRECT_IO must be off, on or auto", .{}); + return error.InvalidArgument; + }; + const dma_block_mib = try envNumber(u32, init.environ_map, "ZML_DMA_BENCH_BLOCK_MIB"); + const memory = std.meta.stringToEnum(zml.io.TensorStore.Memory, init.environ_map.get("ZML_LOAD_MEMORY") orelse "default") orelse { + log.err("ZML_LOAD_MEMORY must be default, host_pinned or host_mapped", .{}); + return error.InvalidArgument; + }; + + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + + var remotes: zml.io.VFS.Remotes = undefined; + try remotes.init(allocator, &vfs, init.environ_map); + defer remotes.deinit(); + + const io = vfs.io(); + + const platform: *zml.Platform = try .auto(allocator, io, try platformOptions(init.environ_map)); + defer platform.deinit(allocator, io); + + // Returns at once: the loader sets its backend up while the + // checkpoint's headers are read and its tensors bound. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ + .checkpoint = path, + .read_parallelism = try envNumber(usize, init.environ_map, "ZML_LOAD_READ_PARALLELISM"), + .dma_block_size = if (dma_block_mib) |mib| @as(usize, mib) * zml.MiB else null, + .direct_io = direct_io, + }); + defer loader.deinit(); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + + const tensor_parallel: *const zml.Mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); + const model_axis: u3 = @intCast(tensor_parallel.resolveLogicalAxis("model").?); + const rows: zml.Sharding.Partitioning = .init(&.{.sharded(model_axis)}); + const columns: zml.Sharding.Partitioning = .init(&.{ .replicated, .sharded(model_axis) }); + + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); + defer store.deinit(); + + const Model = struct { tensors: []zml.Tensor }; + // The checkpoint tensor each one is bound to, at the same index. + const sources = store.registry.tensors.values(); + const model: Model = .{ .tensors = try allocator.alloc(zml.Tensor, sources.len) }; + defer allocator.free(model.tensors); + const view = store.view(); + for (model.tensors, store.registry.tensors.keys(), sources) |*tensor, key, source| { + const rank = source.shape.rank(); + const column_sharded = rank == 2 and + (std.mem.endsWith(u8, key, "o_proj.weight") or std.mem.endsWith(u8, key, "down_proj.weight")); + tensor.* = if (rank == 0 or mode == .replicated) + view.createReplicatedTensor(key, null) + else if (mode == .tp and column_sharded) + view.createTensor(key, null, .tp, columns) + else + view.createTensor(key, null, .tp, rows); + if (memory != .default) view.setMemory(tensor.*, memory); + } + + var progress = std.Progress.start(io, .{ .root_name = "load_check", .disable_printing = true }); + defer progress.end(); + + var buffers = try zml.mem.bufferize(allocator, &model); + defer zml.mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &store, progress); + try loader.wait(); + + if (check_stride != 0) try checkLoaded(allocator, io, sources, buffers.tensors, check_stride); +} + +/// Runs `dma-bench`: the DMA calibration screen on the platform, with the +/// `ZML_DMA_BENCH_*` knobs. +fn dmaBench(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const environ_map = init.environ_map; + + const defaults: zml.io.dma_calibration.Options = .{}; + const window_ms = try envNumber(i64, environ_map, "ZML_DMA_BENCH_WINDOW_MS"); + const opts: zml.io.dma_calibration.Options = .{ + .block_sizes = try envMibList(init.arena.allocator(), environ_map, "ZML_DMA_BENCH_BLOCK_MIB") orelse defaults.block_sizes, + .block_parallelism = try envNumber(usize, environ_map, "ZML_DMA_BENCH_BLOCK_PARALLELISM") orelse defaults.block_parallelism, + .minimum_duration = if (window_ms) |ms| .fromMilliseconds(ms) else defaults.minimum_duration, + .minimum_transfers = try envNumber(u64, environ_map, "ZML_DMA_BENCH_MIN_TRANSFERS") orelse defaults.minimum_transfers, + .block_selection_tolerance = try envNumber(f64, environ_map, "ZML_DMA_BENCH_BLOCK_TOLERANCE") orelse defaults.block_selection_tolerance, + }; + + const platform: *zml.Platform = try .auto(allocator, io, try platformOptions(environ_map)); + defer platform.deinit(allocator, io); + + const block_size = try zml.io.dma_calibration.measure(allocator, io, platform, opts); + log.info("dma_benchmark block_bytes={d} parallelism={d}", .{ block_size, opts.block_parallelism }); +} + +/// The platform settings of the environment, which are not loader knobs. +fn platformOptions(environ_map: *const std.process.Environ.Map) !zml.platform.CreateOptions { + var options: zml.platform.CreateOptions = .{}; + if (try envNumber(u32, environ_map, "ZML_CPU_DEVICE_COUNT")) |device_count| { + options.cpu.device_count = device_count; + } + // Unset keeps the platform's allocator defaults (VMM on CUDA and ROCm); + // set, every XLA GPU target gets a preallocated BFC pool of that + // fraction, whose `bytes_limit` admission measures against. + if (try envNumber(f32, environ_map, "ZML_GPU_MEMORY_FRACTION")) |memory_fraction| { + const pool: zml.platform.CreateOptions.XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }; + options.cuda = pool; + options.rocm = pool; + options.oneapi = pool; + } + return options; +} + +/// Reads every `stride`-th of `buffers` and the largest eligible one back to +/// host and compares the bytes with the checkpoint tensor each was bound +/// to, `sources` at the same index, so a buffer that reported ready before +/// every piece landed is caught. A buffer every device holds whole is +/// compared on every replica; a partitioned one is assembled with +/// `toSliceAlloc`, which keeps the last replica of a region that several +/// devices hold. Each source file is opened once (an `hf://` open is a HEAD +/// plus a redirect). Skipped: sub-byte tensors partitioned over several +/// devices (`toSliceAlloc` places their shards by element stride). +fn checkLoaded( + allocator: std.mem.Allocator, + io: std.Io, + sources: []const zml.safetensors.Tensor, + buffers: []const zml.Buffer, + stride: usize, +) !void { + std.debug.assert(sources.len == buffers.len); + const Eligibility = struct { + fn checkable(source: zml.safetensors.Tensor, buffer: zml.Buffer) bool { + const partitioned = buffer.byteSize() / buffer.numShards() != source.byteSize(); + return !partitioned or source.shape.dtype().bitSizeOf() >= 8; + } + }; + var largest: ?usize = null; + for (sources, buffers, 0..) |source, buffer, i| { + if (!Eligibility.checkable(source, buffer)) continue; + if (largest == null or source.byteSize() > sources[largest.?].byteSize()) largest = i; + } + var files: std.StringHashMapUnmanaged(std.Io.File) = .empty; + defer { + var it = files.valueIterator(); + while (it.next()) |file| file.close(io); + files.deinit(allocator); + } + var expected: []u8 = &.{}; + defer allocator.free(expected); + var replica: []u8 = &.{}; + defer allocator.free(replica); + const started: std.Io.Timestamp = .now(io, .awake); + var checked: usize = 0; + var skipped: usize = 0; + var checked_bytes: usize = 0; + for (sources, buffers, 0..) |source, buffer, i| { + if (i % stride != 0 and i != largest) continue; + if (!Eligibility.checkable(source, buffer)) { + skipped += 1; + continue; + } + const file = files.get(source.file_uri) orelse blk: { + const opened = try std.Io.Dir.openFile(.cwd(), io, source.file_uri, .{ .mode = .read_only }); + errdefer opened.close(io); + try files.put(allocator, source.file_uri, opened); + break :blk opened; + }; + const source_bytes: usize = @intCast(source.byteSize()); + if (expected.len < source_bytes) { + allocator.free(expected); + expected = &.{}; + expected = try allocator.alloc(u8, source_bytes); + } + try source.readRange(io, file, expected[0..source_bytes], 0); + + if (buffer.byteSize() / buffer.numShards() == source_bytes) { + // Every shard holds the whole tensor: compare each replica. + if (replica.len < source_bytes) { + allocator.free(replica); + replica = &.{}; + replica = try allocator.alloc(u8, source_bytes); + } + var shards = buffer.shards(); + var shard_index: usize = 0; + while (shards.next()) |shard| : (shard_index += 1) { + try shard.toHost(io, replica[0..source_bytes]); + try expectSameBytes(source.name, shard_index, expected[0..source_bytes], replica[0..source_bytes]); + checked_bytes += source_bytes; + } + } else { + const slice = try buffer.toSliceAlloc(allocator, io); + defer slice.free(allocator); + try expectSameBytes(source.name, null, expected[0..source_bytes], slice.constData()); + checked_bytes += source_bytes; + } + checked += 1; + } + log.info("load check: ok tensors_checked={d} of {d} skipped={d} bytes={Bi:.2} elapsed={f}", .{ + checked, + sources.len, + skipped, + checked_bytes, + started.untilNow(io, .awake), + }); +} + +fn expectSameBytes(name: []const u8, replica: ?usize, expected: []const u8, actual: []const u8) !void { + if (actual.len == expected.len and std.mem.eql(u8, expected, actual)) return; + const first_bad = std.mem.indexOfDiff(u8, expected, actual) orelse @min(expected.len, actual.len); + log.err("load check: mismatch tensor={s} replica={?d} expected_bytes={d} actual_bytes={d} first_difference={d}", .{ + name, + replica, + expected.len, + actual.len, + first_bad, + }); + return error.LoadContentMismatch; +} + +/// The environment variable `name` as a comma-separated list of MiB, in +/// bytes, null when unset. An entry that does not parse is +/// `InvalidArgument`, logged. +fn envMibList(allocator: std.mem.Allocator, environ_map: *const std.process.Environ.Map, name: []const u8) !?[]const usize { + const text = environ_map.get(name) orelse return null; + var sizes: std.ArrayList(usize) = .empty; + errdefer sizes.deinit(allocator); + var entries = std.mem.tokenizeScalar(u8, text, ','); + while (entries.next()) |entry| { + const mib = std.fmt.parseInt(u32, entry, 10) catch { + log.err("{s} must be MiB counts separated by commas, not '{s}'", .{ name, text }); + return error.InvalidArgument; + }; + try sizes.append(allocator, @as(usize, mib) * zml.MiB); + } + return try sizes.toOwnedSlice(allocator); +} + +/// The environment variable `name` parsed as a `T`, null when unset. A value +/// that does not parse is `InvalidArgument`, logged. +fn envNumber(comptime T: type, environ_map: *const std.process.Environ.Map, name: []const u8) !?T { + const text = environ_map.get(name) orelse return null; + const value = switch (@typeInfo(T)) { + .int => std.fmt.parseInt(T, text, 10), + .float => std.fmt.parseFloat(T, text), + else => @compileError("envNumber: " ++ @typeName(T)), + }; + return value catch { + log.err("{s} must be a number, not '{s}'", .{ name, text }); + return error.InvalidArgument; + }; +} diff --git a/examples/io/main.zig b/examples/io/main.zig index 1cec23b0..e6132ff8 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -3,8 +3,6 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; -const log = std.log.scoped(.vfs); - pub const std_options: std.Options = .{ .log_level = .info, }; @@ -24,34 +22,12 @@ pub fn main(init: std.process.Init) !void { const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; const path = it.next() orelse return error.MissingPath; - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - - try http_client.initDefaultProxies(allocator, init.environ_map); - defer http_client.deinit(); - - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); - defer vfs_https.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); - defer gcs_vfs.deinit(); - var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.register("file", vfs_file.io()); - try vfs.register("https", vfs_https.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); - try vfs.register("gs", gcs_vfs.io()); + var remotes: zml.io.VFS.Remotes = undefined; + try remotes.init(allocator, &vfs, init.environ_map); + defer remotes.deinit(); const io = vfs.io(); @@ -132,7 +108,7 @@ pub fn main(init: std.process.Init) !void { const root = try TensorNode.init(init.arena.allocator(), ""); - var registry_it = registry.iterator(); + var registry_it = registry.tensors.iterator(); while (registry_it.next()) |kv| { const name = kv.key_ptr.*; const tensor = kv.value_ptr.*; @@ -162,6 +138,11 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); + // Returns at once: the loader sets its backend up while the + // checkpoint's headers are read and its tensors bound. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); + defer loader.deinit(); + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); defer registry.deinit(); @@ -171,58 +152,36 @@ pub fn main(init: std.process.Init) !void { ); const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); + // The store takes the registry's entries. + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); defer store.deinit(); const AllTensorsModel = struct { tensors: []zml.Tensor, }; - const tensor_count = registry.tensors.count(); - - const tensors = try allocator.alloc(zml.Tensor, tensor_count); + const tensors = try allocator.alloc(zml.Tensor, store.registry.tensors.count()); defer allocator.free(tensors); - var registry_it = registry.iterator(); - var load_count: usize = 0; - while (registry_it.next()) |entry| : (load_count += 1) { - tensors[load_count] = switch (mesh_type) { - .replicated => store.view().createReplicatedTensor(entry.key_ptr.*, null), - .sharded => if (entry.value_ptr.shape.rank() > 0) - store.view().createTensor(entry.key_ptr.*, null, .tp, tp_partitioning) + for (tensors, store.registry.tensors.keys(), store.registry.tensors.values()) |*tensor, key, entry| { + tensor.* = switch (mesh_type) { + .replicated => store.view().createReplicatedTensor(key, null), + .sharded => if (entry.shape.rank() > 0) + store.view().createTensor(key, null, .tp, tp_partitioning) else - store.view().createReplicatedTensor(entry.key_ptr.*, null), + store.view().createReplicatedTensor(key, null), }; } const model: AllTensorsModel = .{ .tensors = tensors }; var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); - progress.increaseEstimatedTotalItems(load_count); defer progress.end(); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, AllTensorsModel, &model); - defer { - for (buffers.tensors) |*b| b.deinit(); - allocator.free(buffers.tensors); - } - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .parallelism = 8, - .dma_chunks = 16, - .dma_chunk_size = 256 * zml.MiB, - }); - defer loader.deinit(); - - try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + var buffers = try zml.mem.bufferize(allocator, &model); + defer zml.mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &store, progress); + try loader.wait(); // logs "Loaded weights" }, } } diff --git a/examples/llm/main.zig b/examples/llm/main.zig index b24c022a..b5c8f3ec 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -53,28 +53,12 @@ pub fn main(init: std.process.Init) !void { // // Virtual File Systems // - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - defer http_client.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); - defer gcs_vfs.deinit(); - var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.register("file", vfs_file.io()); - try vfs.register("gs", gcs_vfs.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); + var remotes: zml.io.VFS.Remotes = undefined; + try remotes.init(allocator, &vfs, init.environ_map); + defer remotes.deinit(); const io = vfs.io(); @@ -107,7 +91,7 @@ pub fn main(init: std.process.Init) !void { // Defines how the model's tensors are sharded across the available devices. const meshes: models.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer store.deinit(); const generation: models.GenerationOptions = .{ @@ -116,9 +100,6 @@ pub fn main(init: std.process.Init) !void { }, }; - var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); - defer model.deinit(allocator); - // // Load the model and compile it // @@ -128,14 +109,29 @@ pub fn main(init: std.process.Init) !void { var tokenizer = try loadTokenizer(allocator, io, repo, &progress); defer tokenizer.deinit(); + var model: models.LoadedModel = undefined; var compiled_model = try allocator.create(models.CompiledModel); defer allocator.destroy(compiled_model); - compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); - defer compiled_model.deinit(); + var model_buffers = buffers: { + // Returns at once: the loader sets its backend up on a task of its + // own while the model is initialized and compiled. It is freed at + // the end of this block, with its pinned host memory, before the + // chat starts. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = args.model }); + defer loader.deinit(); - // Load buffers after the model compilation to be sure to give enough room to the autotune. - var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); - defer model.unloadBuffers(&model_buffers, allocator); + model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); + errdefer model.deinit(allocator); + + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); + errdefer compiled_model.deinit(); + + // Load buffers after the model compilation to be sure to give enough room to the autotune. + break :buffers try models.LoadedModel.loadBuffers(&model, allocator, &loader, progress, &store); + }; + defer model.deinit(allocator); + defer compiled_model.deinit(); + defer zml.mem.deinitBufferized(allocator, &model_buffers); progress.end(); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 20aa60c7..33e22bf3 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -55,26 +55,23 @@ pub const LoadedModel = union(ModelType) { } } + /// Loads the model's weights from the store with `loader`, which logs + /// them. Free them with `zml.mem.deinitBufferized`. pub fn loadBuffers( - self: *LoadedModel, + self: *const LoadedModel, allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, + loader: *zml.io.Loader, + progress: std.Progress.Node, + store: *const zml.io.TensorStore, ) !Buffers { - return switch (self.*) { - inline else => |*m, t| @unionInit( - Buffers, - @tagName(t), - try m.loadBuffers(allocator, io, platform, store, progress), - ), - }; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { switch (self.*) { - inline else => |*loaded, t| loaded.unloadBuffers(&@field(buffers, @tagName(t)), allocator), + inline else => |*m, t| { + var buffers = try zml.mem.bufferize(allocator, &m.inner); + errdefer zml.mem.deinitBufferized(allocator, &buffers); + try loader.start(&m.inner, &buffers, store, progress); + try loader.wait(); + return @unionInit(Buffers, @tagName(t), buffers); + }, } } diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index dd21c7f9..bee197fa 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -6,8 +6,6 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.lfm); - pub const Config = struct { architectures: []const []const u8, block_auto_adjust_ff_dim: bool, @@ -72,43 +70,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - Model.unloadBuffers(buffers, allocator); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -164,47 +125,6 @@ pub const Model = struct { }; } - pub fn loadBuffers( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 8, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - LmHead.unloadBuffers(&self.lm_head); - for (self.layers) |*layer| { - DecoderLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - pub fn forward( self: Model, tokens: zml.Tensor, @@ -324,10 +244,6 @@ pub const LmHead = struct { const tokens = new_tokens.convert(.u32); return .{ .tokens = if (input.tokens.dim(.seq) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng }; } - - pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { - RmsNorm.unloadBuffers(&self.embedding_norm); - } }; pub const OperatorKind = enum { conv, full_attention }; @@ -426,16 +342,6 @@ pub const DecoderLayer = struct { .kv_cache_index = kv_cache_index.reuseBuffer(input.kv_cache_index), }; } - - pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { - switch (self.operator) { - .conv => |*operator| ShortConv.unloadBuffers(operator), - .self_attn => |*operator| Attention.unloadBuffers(operator), - } - RmsNorm.unloadBuffers(&self.operator_norm); - RmsNorm.unloadBuffers(&self.ffn_norm); - Mlp.unloadBuffers(&self.feed_forward); - } }; pub const ConvParameters = struct { is_prefill: bool }; @@ -501,12 +407,6 @@ pub const ShortConv = struct { const output = self.out_proj.forward(y); return .{ output.reuseBuffer(input), cache.reuseBuffer(cache_) }; } - - pub fn unloadBuffers(self: *zml.Bufferized(ShortConv)) void { - Linear.unloadBuffers(&self.in_proj); - Linear.unloadBuffers(&self.out_proj); - self.kernel.deinit(); - } }; pub const Attention = struct { @@ -594,15 +494,6 @@ pub const Attention = struct { return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; } - - pub fn unloadBuffers(self: *zml.Bufferized(Attention)) void { - Linear.unloadBuffers(&self.q_proj); - Linear.unloadBuffers(&self.k_proj); - Linear.unloadBuffers(&self.v_proj); - Linear.unloadBuffers(&self.out_proj); - RmsNorm.unloadBuffers(&self.q_layernorm); - RmsNorm.unloadBuffers(&self.k_layernorm); - } }; pub const Cache = struct { @@ -613,11 +504,6 @@ pub const Cache = struct { return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; } - pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { - ConvCache.unloadBuffers(&self.conv); - KvCache.unloadBuffers(&self.kv); - } - pub fn reuseBuffer(self: Cache, other: Cache) Cache { return .{ .conv = self.conv.reuseBuffer(other.conv), .kv = self.kv.reuseBuffer(other.kv) }; } @@ -638,10 +524,6 @@ pub const ConvCache = struct { return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; } - pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { - self.state.deinit(); - } - pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { return .{ .state = self.state.reuseBuffer(other.state) }; } @@ -660,11 +542,6 @@ pub const KvCache = struct { return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; } - pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { - self.k.deinit(); - self.v.deinit(); - } - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { return .{ .k = self.k.reuseBuffer(other.k), .v = self.v.reuseBuffer(other.v) }; } @@ -700,11 +577,6 @@ pub const Linear = struct { return .{ .weight = weight, .bias = bias, .in_tag = zml.Shape.toTag(tag), .out_tag = out_tag }; } - pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { - linear.weight.deinit(); - if (linear.bias) |*bias| bias.deinit(); - } - pub fn forward(self: Linear, x: zml.Tensor) zml.Tensor { var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; @@ -724,12 +596,6 @@ const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - Linear.unloadBuffers(&self.w1); - Linear.unloadBuffers(&self.w2); - Linear.unloadBuffers(&self.w3); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); } @@ -744,10 +610,6 @@ const RmsNorm = struct { return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index ecc5abf8..df617bec 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -31,7 +31,7 @@ pub const Session = struct { ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); - errdefer model.Cache.unloadBuffers(&cache_buffers); + errdefer zml.mem.deinitBufferized(allocator, &cache_buffers); var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); @@ -62,7 +62,7 @@ pub const Session = struct { pub fn deinit(self: *Session) void { self.prefill.deinit(self.allocator); self.decode.deinit(self.allocator); - model.Cache.unloadBuffers(&self.cache_buffers); + zml.mem.deinitBufferized(self.allocator, &self.cache_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buf); self.generated_token_slice.free(self.allocator); } diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 19b28df6..4934d3e6 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -41,7 +41,7 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer store.deinit(); var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); @@ -49,8 +49,12 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); - defer repo_model.unloadBuffers(&model_buffers, allocator); + var loader: zml.io.Loader = try .init(allocator, io, platform, .{}); + defer loader.deinit(); + var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); + defer zml.mem.deinitBufferized(allocator, &model_buffers); + try loader.start(&repo_model.inner, &model_buffers, &store, progress); + try loader.wait(); const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { .stablehlo => .{ .stablehlo = {} }, @@ -85,7 +89,7 @@ pub fn run( var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var activation_store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer activation_store.deinit(); var ctx = TestContext{ @@ -175,7 +179,7 @@ const TestContext = struct { var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - exe.call(self.io, args, &res, .{}); + try exe.call(self.io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); @@ -304,16 +308,12 @@ const TestContext = struct { }; fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { - const shape = store.view().getShape(key) orelse return error.NotFound; + const tensor = store.view().get(key) orelse return error.NotFound; - const host_bytes = try allocator.alloc(u8, shape.byteSize()); + const host_bytes = try allocator.alloc(u8, tensor.byteSize()); defer allocator.free(host_bytes); - var io_buffer: [8 * 1024]u8 = undefined; - var reader = try store.view().getReader(key, io, &io_buffer); - defer reader.deinit(); + try tensor.read(io, host_bytes); - _ = try reader.interface.readSliceAll(host_bytes); - - return zml.Buffer.fromBytes(io, platform, shape, host_bytes); + return zml.Buffer.fromBytes(io, platform, tensor.shape, host_bytes); } diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 05610dc1..0dfcbb18 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -6,8 +6,6 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.llama); - pub const Config = struct { bos_token_id: u32, eos_token_id: EosTokens, @@ -69,43 +67,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - if (buffers.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); - Llama.unloadBuffers(&buffers.model, allocator); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -161,43 +122,6 @@ pub const Model = struct { self.model.deinit(allocator); } - pub fn loadBuffers( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - if (self.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); - Llama.unloadBuffers(&self.model, allocator); - } - fn lmHead(self: Model) LmHead { return .init(self); } @@ -270,15 +194,6 @@ const Llama = struct { allocator.free(self.layers); } - pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - RmsNorm.unloadBuffers(&self.norm); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. pub fn forward( @@ -396,13 +311,6 @@ pub const TransformerLayer = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - SelfAttn.unloadBuffers(&self.self_attn); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - Mlp.unloadBuffers(&self.mlp); - } - pub const Input = struct { layer: TransformerLayer, hidden: zml.Tensor, @@ -461,10 +369,6 @@ const RmsNorm = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); @@ -486,10 +390,6 @@ const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const proj = self.up_proj.forward(x, x.dtype()); var output = self.gate_proj.forward(x, x.dtype()); @@ -531,10 +431,6 @@ const SelfAttn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - /// Self zml.attention. /// - If token_index is set, x is assumed to be the representation of one new token, /// and kv_cache will be read for the previous tokens. diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 1b5894f1..dae0d054 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -39,7 +39,7 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer store.deinit(); var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); @@ -47,8 +47,12 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); - defer repo_model.unloadBuffers(&model_buffers, allocator); + var loader: zml.io.Loader = try .init(allocator, io, platform, .{}); + defer loader.deinit(); + var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); + defer zml.mem.deinitBufferized(allocator, &model_buffers); + try loader.start(&repo_model.inner, &model_buffers, &store, progress); + try loader.wait(); progress.end(); try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); @@ -66,7 +70,7 @@ fn run( var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); + var activation_store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = meshes }); defer activation_store.deinit(); try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); @@ -98,7 +102,7 @@ fn testLayer( ) !void { const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); defer allocator.free(in_key); - const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; + const in_shape = (activation_store.get(in_key) orelse return error.NotFound).shape; var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_shape); @@ -126,7 +130,7 @@ fn testLayer( var res = try exe.results(allocator); defer res.deinit(allocator); - exe.call(io, args, &res, .{}); + try exe.call(io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); @@ -134,16 +138,12 @@ fn testLayer( } fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { - const shape = store.getShape(key) orelse return error.NotFound; + const tensor = store.get(key) orelse return error.NotFound; - const host_bytes = try allocator.alloc(u8, shape.byteSize()); + const host_bytes = try allocator.alloc(u8, tensor.byteSize()); defer allocator.free(host_bytes); - var io_buffer: [8 * 1024]u8 = undefined; - var reader = try store.getReader(key, io, &io_buffer); - defer reader.deinit(); + try tensor.read(io, host_bytes); - _ = try reader.interface.readSliceAll(host_bytes); - - return zml.Buffer.fromBytes(io, platform, shape, host_bytes); + return zml.Buffer.fromBytes(io, platform, tensor.shape, host_bytes); } diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 77d3884e..a434c489 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -6,8 +6,6 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.qwen3_5); - pub const Config = struct { text_config: TextConfig, }; @@ -74,44 +72,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - TextModel.unloadBuffers(&buffers.text_model, allocator); - zml.nn.Linear.unloadBuffers(&buffers.lm_head); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -153,7 +113,7 @@ pub const Model = struct { pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer const lm_head_prefix = b: { - if (store.hasKey("lm_head.weight")) break :b "lm_head"; + if (store.get("lm_head.weight") != null) break :b "lm_head"; break :b "model.language_model.embed_tokens"; }; return .{ @@ -172,43 +132,6 @@ pub const Model = struct { self.text_model.deinit(allocator); } - pub fn load( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - meshes: []const *const zml.Mesh, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store, meshes); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - TextModel.unloadBuffers(&self.text_model, allocator); - zml.nn.Linear.unloadBuffers(&self.lm_head); - } pub fn forward( self: Model, tokens_: zml.Tensor, @@ -322,15 +245,6 @@ pub const TextModel = struct { allocator.free(self.layers); } - pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); - } - pub fn forward( self: TextModel, tokens: zml.Tensor, @@ -401,16 +315,6 @@ pub const TransformerLayer = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - switch (self.attn) { - .full_attention => |*self_attn| SelfAttn.unloadBuffers(self_attn), - .linear_attention => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), - } - Mlp.unloadBuffers(&self.mlp); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - } - pub fn forward( self: TransformerLayer, x0: zml.Tensor, @@ -522,10 +426,6 @@ pub const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const up_projed = self.up_proj.forward(x, x.dtype()); const gate = self.gate_proj.forward(x, x.dtype()); @@ -577,10 +477,6 @@ pub const SelfAttn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const q_proj = self.q_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); const q, var gate = q_proj.chunkExact(.hd, 2); @@ -798,10 +694,6 @@ pub const GatedDeltaNet = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - zml.Buffer.deinitAll(GatedDeltaNet, self); - } - fn recurrentGatedDeltaRule( query: zml.Tensor, key: zml.Tensor, @@ -977,10 +869,6 @@ pub const RmsNorm = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const weight_f32 = self.weight.convert(.f32); @@ -998,10 +886,6 @@ pub const RmsNormGated = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 11ce8151..9fa5300f 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -100,43 +100,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - TextModel.unloadBuffers(&buffers.text_model, allocator); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -188,42 +151,6 @@ pub const Model = struct { self.text_model.deinit(allocator); } - pub fn load( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - meshes: []const *const zml.Mesh, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store, meshes); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - TextModel.unloadBuffers(&self.text_model, allocator); - } pub fn forward( self: Model, tokens_: zml.Tensor, @@ -305,7 +232,7 @@ pub const TextModel = struct { config: Config, gen_options: Model.GenOptions, ) !TextModel { - const lm_head_prefix = if (store.root().hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; + const lm_head_prefix = if (store.root().get("lm_head.weight") != null) "lm_head" else "model.language_model.embed_tokens"; const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); errdefer allocator.free(layers); @@ -331,16 +258,6 @@ pub const TextModel = struct { allocator.free(self.layers); } - pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer, allocator); - } - allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); - zml.nn.Linear.unloadBuffers(&self.lm_head); - } - pub fn sampler(self: TextModel) Sampler { return .{ .norm = self.norm, @@ -437,16 +354,6 @@ pub const TransformerLayer = struct { self.moe.deinit(allocator); } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer), allocator: std.mem.Allocator) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - switch (self.attn) { - .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), - .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), - } - Moe.unloadBuffers(&self.moe, allocator); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - } - pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { const self = input.layer; const x0 = input.hidden; @@ -582,10 +489,6 @@ pub const SelfAttn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const q_proj = self.q_proj.forward(x, x.dtype()) .splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); @@ -688,10 +591,6 @@ pub const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const up_projed = self.up_proj.forward(x, x.dtype()); const gate = self.gate_proj.forward(x, x.dtype()); @@ -709,18 +608,14 @@ const Router = struct { pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { return .{ .router = .init( - store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), - store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), + store.createTensor("weight", .{ .expert, .d }, .replicated, .{}), + store.maybeCreateTensor("bias", .{.expert}, .replicated, .{}), .d, ), .num_experts_per_tok = num_experts_per_tok, }; } - pub fn unloadBuffers(self: *zml.Bufferized(Router)) void { - zml.Buffer.deinitAll(Router, self); - } - pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const router_logits = self.router.forward(x, .f32); const routing = router_logits.topK(.{ .topk = .expert }, self.num_experts_per_tok, .{}); @@ -756,8 +651,8 @@ pub const Moe = struct { return .{ .shared_expert = Mlp.init(store.withPrefix("shared_expert")), .shared_expert_gate = .init( - store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), - store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{}), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{}), .d, ), .gate_up_proj = .init(gate_up_proj_tensor, null, .d), @@ -797,11 +692,6 @@ pub const Moe = struct { return moe_output.add(shared); } - - pub fn unloadBuffers(self: *zml.Bufferized(Moe), allocator: std.mem.Allocator) void { - _ = allocator; - zml.Buffer.deinitAll(Moe, self); - } }; pub const TextRotaryEmbedding = struct { @@ -927,10 +817,6 @@ pub const GatedDeltaNet = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - zml.Buffer.deinitAll(GatedDeltaNet, self); - } - fn recurrentGatedDeltaRule( query: zml.Tensor, key: zml.Tensor, @@ -1096,10 +982,6 @@ pub const RmsNorm = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const weight_f32 = self.weight.convert(.f32); @@ -1117,10 +999,6 @@ pub const RmsNormGated = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index f5842af8..878b269b 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -37,32 +37,6 @@ const Mnist = struct { }; } - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, Mnist, self); - errdefer unloadBuffers(&buffers); - - var loader: zml.io.Loader = try .init(allocator, platform, .default); - errdefer loader.deinit(); - - try loader.load(io, Mnist, self, &buffers, store, .{}); - try loader.await(io); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { - self.fc1.weight.deinit(); - self.fc1.bias.deinit(); - self.fc2.weight.deinit(); - self.fc2.bias.deinit(); - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); @@ -80,24 +54,32 @@ const Mnist = struct { pub fn main(init: std.process.Init) !void { const arena = init.arena; const allocator = init.gpa; - const io = init.io; // Parse program args const process_args = try init.minimal.args.toSlice(arena.allocator()); const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Read model shapes. - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); - defer registry.deinit(); + // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + const io = vfs.io(); // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); - // Init model - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + // Returns at once: the loader calibrates while the model is compiled. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = model_path }); + defer loader.deinit(); + + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); defer store.deinit(); + + // Init model const mnist_model: Mnist = .init(store.view()); // // Compile model @@ -110,16 +92,11 @@ pub fn main(init: std.process.Init) !void { }; defer exe.deinit(); - // Load buffers - var mnist_buffers = blk: { - log.info("Transfering weights....", .{}); - const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{f}]", .{ - start.untilNow(io, .awake), - }); - break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); - }; - defer Mnist.unloadBuffers(&mnist_buffers); + // Load weights + var mnist_buffers = try zml.mem.bufferize(allocator, &mnist_model); + defer zml.mem.deinitBufferized(allocator, &mnist_buffers); + try loader.start(&mnist_model, &mnist_buffers, &store, .none); + try loader.wait(); // logs "Loaded weights" var args = try exe.args(allocator); defer args.deinit(allocator); @@ -147,7 +124,7 @@ pub fn main(init: std.process.Init) !void { printDigit(&sample); args.set(.{ mnist_buffers, input_buffer }); - exe.call(io, args, &results, .{}); + try exe.call(io, args, &results, .{}); var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index a539ae5a..76684d94 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -31,25 +31,6 @@ const Mnist = struct { }; } - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, Mnist, self); - errdefer zml.Buffer.deinitAll(Mnist, &buffers); - - var loader: zml.io.Loader = try .init(allocator, platform, .default); - errdefer loader.deinit(); - - try loader.load(io, Mnist, self, &buffers, store, .{}); - try loader.await(io); - - return buffers; - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); @@ -78,16 +59,16 @@ const Mnist = struct { pub fn main(init: std.process.Init) !void { const arena = init.arena; const allocator = init.gpa; - const io = init.io; // Parse program args const process_args = try init.minimal.args.toSlice(arena.allocator()); const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Read model shapes. - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); - defer registry.deinit(); + // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + const io = vfs.io(); // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); @@ -102,10 +83,17 @@ pub fn main(init: std.process.Init) !void { )); log.info("topology used: {f}", .{dp_mp}); + // Returns at once: the loader calibrates while the model is compiled. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = model_path }); + defer loader.deinit(); + var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); defer profiler.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{dp_mp} }); defer store.deinit(); const mnist_model: Mnist = .init(store.view()); @@ -131,16 +119,11 @@ pub fn main(init: std.process.Init) !void { }; defer exe.deinit(); - // Load buffers - var mnist_buffers = blk: { - log.info("Transfering weights....", .{}); - const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{f}]", .{ - start.untilNow(io, .awake), - }); - break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); - }; - defer zml.Buffer.deinitAll(Mnist, &mnist_buffers); + // Load weights + var mnist_buffers = try zml.mem.bufferize(allocator, &mnist_model); + defer zml.mem.deinitBufferized(allocator, &mnist_buffers); + try loader.start(&mnist_model, &mnist_buffers, &store, .none); + try loader.wait(); // logs "Loaded weights" var args = try exe.args(allocator); defer args.deinit(allocator); @@ -176,7 +159,7 @@ pub fn main(init: std.process.Init) !void { args.set(.{ mnist_buffers, input_buffer }); try profiler.start(); - exe.call(io, args, &results, .{}); + try exe.call(io, args, &results, .{}); if (try profiler.stop()) |report| { log.info("Wrote profiler files to {s} and {s}", .{ report.protobuf_path, report.perfetto_path }); } diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index cc46a1bd..29de9597 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -242,7 +242,7 @@ pub fn main(init: std.process.Init) !void { defer exe_results.deinit(allocator); exe_args.set(.{ w_buf, b_buf, input_buf }); - exe.call(io, exe_args, &exe_results, .{}); + try exe.call(io, exe_args, &exe_results, .{}); var out = exe_results.get(zml.Buffer); defer out.deinit(); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 3715361f..499e753c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -484,11 +484,6 @@ pub const Client = opaque { return @ptrCast(ret.executable.?); } - pub fn compile(self: *const Client, api: *const Api, io: std.Io, args: CompileArgs) ApiError!*LoadedExecutable { - var future = io.async(compileRaw, .{ self, api, args }); - return future.await(io); - } - pub const BufferFromHostBufferArgs = struct { data: [*]const u8, buffer_type: BufferType, @@ -677,8 +672,19 @@ pub const Client = opaque { }; pub fn createUninitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { + return self.createUninitializedBufferThrough(api, args, Api.call); + } + + /// As `createUninitializedBuffer`, for a request the caller expects to + /// fail: its error is logged at debug level only. + pub fn createUninitializedBufferSilent(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { + return self.createUninitializedBufferThrough(api, args, Api.callSilent); + } + + /// `call` is `Api.call` or `Api.callSilent`. + fn createUninitializedBufferThrough(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs, comptime call: anytype) ApiError!*Buffer { var layout = args.layout.toCStruct(); - const ret = try api.call(.PJRT_Client_CreateUninitializedBuffer, .{ + const ret = try call(api, .PJRT_Client_CreateUninitializedBuffer, .{ .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = args.dims.len, diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index ee31bd83..6fcda16b 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -74,6 +74,142 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_rocm.so" }); - break :blk .loadFrom(lib_path); + const api = try pjrt.Api.loadFrom(lib_path); + loaded_hip = Hip.loaded(); + break :blk api; }; } + +/// Whether `releaseRuntimeDeviceMemory` asks the devices for their whole +/// memory. The first call that finds the HIP runtime missing, or a device +/// that does not refuse, logs it and clears this for good. It lives here, +/// not with a PJRT client: the HIP runtime the plugin loaded, and the +/// memory it keeps, belong to the process and serve every client. +var releases_runtime_memory: std.atomic.Value(bool) = .init(true); +/// The HIP calls of `releaseRuntimeDeviceMemory`, resolved by `load`, which +/// zml calls once per process; null when `libamdhip64.so.7` is not loaded +/// or lacks one of them. +var loaded_hip: ?Hip = null; + +/// Makes the HIP runtime the plugin loaded give back the device memory it +/// keeps outside the plugin's allocator, chiefly the scratch of kernels +/// that spill registers, which ROCr keeps bound to its queues and in its +/// caches until the driver refuses one of its allocations. It asks every +/// device in turn for its whole memory, rounded down to 2 MiB: the driver +/// refuses, ROCr reclaims the scratch of the device's queues and frees its +/// caches, and HIP answers `hipErrorOutOfMemory`. Kernels that spill later +/// map their scratch again. Every queue in the process that holds scratch +/// on the device is suspended and resumed, and the call waits for its +/// kernels using scratch while holding the device's ROCr memory lock: +/// running work and allocations on that device stall meanwhile, a stall, +/// not a deadlock. Restores the thread's HIP device, and clears any HIP +/// error left on the thread, earlier ones included. +/// +/// Returns whether every device refused. A missing `libamdhip64.so.7` or +/// symbol, or a device that does not refuse (a grant is freed at once), is +/// logged once and ends the trim for good: the remaining devices are not +/// asked, nor any device in later calls. +pub fn releaseRuntimeDeviceMemory(hip_ordinals: []const c_int) bool { + if (comptime !isEnabled() or builtin.os.tag != .linux) return false; + if (!releases_runtime_memory.load(.monotonic)) return false; + const hip = loaded_hip orelse { + if (releases_runtime_memory.swap(false, .monotonic)) { + log.warn("The HIP runtime (libamdhip64.so.7) is not loaded or lacks a symbol; the device memory it keeps is no longer given back", .{}); + } + return false; + }; + + var previous: c_int = 0; + const has_previous = hip.hipGetDevice(&previous) == hip_success; + defer { + if (has_previous) _ = hip.hipSetDevice(previous); + _ = hip.hipGetLastError(); + } + + for (hip_ordinals) |ordinal| { + const answer = hip.requestWholeDevice(ordinal); + if (answer == .refused) continue; + if (releases_runtime_memory.swap(false, .monotonic)) { + log.warn("HIP device {d} did not refuse a request for its whole memory: {f}; the device memory the HIP runtime keeps is no longer given back", .{ ordinal, answer }); + } + return false; + } + return true; +} + +/// `hipError_t` values (hip_runtime_api.h). +const hip_success: c_int = 0; +const hip_error_out_of_memory: c_int = 2; + +/// The HIP calls `releaseRuntimeDeviceMemory` makes, from the HIP runtime +/// already in the process. +const Hip = struct { + hipGetDevice: *const fn (device: *c_int) callconv(.c) c_int, + hipSetDevice: *const fn (device: c_int) callconv(.c) c_int, + hipMemGetInfo: *const fn (free: *usize, total: *usize) callconv(.c) c_int, + hipMalloc: *const fn (ptr: *?*anyopaque, size: usize) callconv(.c) c_int, + hipFree: *const fn (ptr: ?*anyopaque) callconv(.c) c_int, + hipGetLastError: *const fn () callconv(.c) c_int, + + /// Only finds the library: it never loads it, nor unloads it (see the + /// note on thread-local storage in `load`). `libzml_rocm.so` needs + /// `libamdhip64.so.7`, and the plugin's own dlopen of `libamdhip64.so` + /// resolves to the same file (`zmlxrocm_dlopen`). + fn loaded() ?Hip { + var lib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen("libamdhip64.so.7", .{ .LAZY = true, .NOLOAD = true }) orelse return null, + } }; + var hip: Hip = undefined; + inline for (@typeInfo(Hip).@"struct".field_names) |name| { + @field(hip, name) = lib.lookup(@FieldType(Hip, name), name) orelse return null; + } + return hip; + } + + /// Asks HIP for the whole memory of the device, made the current one, + /// when less than that is free, so that the driver cannot grant it. + /// Frees what HIP grants anyway. + fn requestWholeDevice(hip: Hip, ordinal: c_int) Answer { + const set_device = hip.hipSetDevice(ordinal); + if (set_device != hip_success) return .{ .set_device = set_device }; + var free: usize = 0; + var total: usize = 0; + const mem_get_info = hip.hipMemGetInfo(&free, &total); + if (mem_get_info != hip_success) return .{ .mem_get_info = mem_get_info }; + // ROCr rounds a request up to 2 MiB and refuses one beyond the + // device's size itself, without the driver's refusal that trims. + const request = std.mem.alignBackward(usize, total, 2 << 20); + if (free >= request) return .{ .not_asked = .{ .free = free, .request = request } }; + var ptr: ?*anyopaque = null; + const answer = hip.hipMalloc(&ptr, request); + if (answer == hip_error_out_of_memory) return .refused; + if (answer == hip_success) _ = hip.hipFree(ptr); + return .{ .malloc = answer }; + } +}; + +/// How a device answered `Hip.requestWholeDevice`. +const Answer = union(enum) { + /// hipMalloc answered `hipErrorOutOfMemory`, as expected. + refused, + /// So much is free that the driver could grant the request. + not_asked: struct { free: usize, request: usize }, + /// hipMalloc's other answers: `hip_success` for a grant, freed at once. + malloc: c_int, + /// The `hipError_t` of hipSetDevice or hipMemGetInfo, which failed. + set_device: c_int, + mem_get_info: c_int, + + pub fn format(answer: Answer, writer: *std.Io.Writer) std.Io.Writer.Error!void { + switch (answer) { + .refused => try writer.writeAll("refused"), + .not_asked => |memory| try writer.print("not asked, {d} bytes being free for a request of {d}", .{ memory.free, memory.request }), + .malloc => |code| if (code == hip_success) + try writer.writeAll("granted, then freed") + else + try writer.print("hipMalloc returned hipError_t {d}", .{code}), + .set_device => |code| try writer.print("hipSetDevice returned hipError_t {d}", .{code}), + .mem_get_info => |code| try writer.print("hipMemGetInfo returned hipError_t {d}", .{code}), + } + } +}; diff --git a/stdx/Io.zig b/stdx/Io.zig index 5b5a35d8..453ee503 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -22,130 +22,20 @@ pub const Dir = struct { } }; -pub const LimitedGroup = struct { - limit: usize, - in_flight: std.atomic.Value(usize) = .init(0), - group: std.Io.Group = .init, - cond: std.Io.Condition = .init, - mutex: std.Io.Mutex = .init, - - fn Wrapper(comptime function: anytype) type { - return struct { - fn wrapper(self: *LimitedGroup, io: std.Io, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.Cancelable!void { - while (true) { - var in_flight = self.in_flight.load(.acquire); - while (in_flight < self.limit) { - if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { - in_flight = actual; - continue; - } - - defer { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - _ = self.in_flight.fetchSub(1, .release); - self.cond.signal(io); - } - return @call(.auto, function, args); - } - - try self.mutex.lock(io); - defer self.mutex.unlock(io); - while (self.in_flight.load(.acquire) >= self.limit) { - try self.cond.wait(io, &self.mutex); - } - } - } - }; - } - - pub fn init(limit: usize) LimitedGroup { - return .{ .limit = limit }; - } - - pub fn async(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void { - self.group.async(io, Wrapper(function).wrapper, .{ self, io, args }); - } - - pub fn concurrent(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.ConcurrentError!void { - try self.group.concurrent(io, Wrapper(function).wrapper, .{ self, io, args }); - } - - pub fn await(self: *LimitedGroup, io: std.Io) std.Io.Cancelable!void { - try self.group.await(io); - } - - pub fn cancel(self: *LimitedGroup, io: std.Io) void { - self.group.cancel(io); - } -}; - -pub const AllocatingLimitedConcurrentGroup = struct { - allocator: std.mem.Allocator, - - queue_buffer: []bool = &[_]bool{}, - limit: usize, - - group: std.Io.Group = .init, - queue: std.Io.Queue(bool) = undefined, - - pub fn init(allocator: std.mem.Allocator, limit: usize) std.mem.Allocator.Error!AllocatingLimitedConcurrentGroup { - const buffer = try allocator.alloc(bool, limit); - @memset(buffer, false); - var self: AllocatingLimitedConcurrentGroup = .{ - .allocator = allocator, - .queue_buffer = buffer, - .limit = limit, - .group = .init, - .queue = undefined, - }; - self.queue = std.Io.Queue(bool).init(self.queue_buffer); - return self; - } - - pub fn deinit(self: *AllocatingLimitedConcurrentGroup) void { - self.allocator.free(self.queue_buffer); - } - - pub fn concurrent( - self: *AllocatingLimitedConcurrentGroup, - io: std.Io, - comptime function: anytype, - args: std.meta.ArgsTuple(@TypeOf(function)), - ) !void { - self.queue.putOneUncancelable(io, true) catch unreachable; - - try self.group.concurrent(io, struct { - fn wrapper( - io_: std.Io, - queue: *std.Io.Queue(bool), - inner_args: std.meta.ArgsTuple(@TypeOf(function)), - ) !void { - defer _ = queue.getOneUncancelable(io_) catch unreachable; - return @call(.auto, function, inner_args); - } - }.wrapper, .{ io, &self.queue, args }); - } - - pub fn await(self: *AllocatingLimitedConcurrentGroup, io: std.Io) std.Io.Cancelable!void { - try self.group.await(io); - } - - pub fn cancel(self: *AllocatingLimitedConcurrentGroup, io: std.Io) void { - self.group.cancel(io); - } -}; - pub const Duration = struct { pub fn div(self: std.Io.Duration, rhs: u64) std.Io.Duration { return .fromNanoseconds(@divTrunc(self.nanoseconds, @as(i96, @intCast(rhs)))); } pub fn hz(self: std.Io.Duration) u64 { - return @intCast(std.time.ns_per_s / self.nanoseconds); + return @intCast(@divTrunc(std.time.ns_per_s, self.nanoseconds)); } pub fn hzFloat(self: std.Io.Duration) f64 { return (1 * std.time.ns_per_s) / @as(f64, @floatFromInt(self.nanoseconds)); } }; + +test "Duration.hz" { + try std.testing.expectEqual(@as(u64, 4), Duration.hz(.fromMilliseconds(250))); +} diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 1b19e638..5ed0b048 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -87,6 +87,33 @@ pub fn Once(comptime f: anytype) type { }; } +/// The first error of concurrent tasks, kept once and readable by any of +/// them. The state is the error's integer value, zero while none is kept. +pub fn FirstError(comptime E: type) type { + return struct { + code: std.atomic.Value(@Int(.unsigned, @bitSizeOf(anyerror))) = .init(0), + + /// Keeps `err` unless an error is kept already; true when this call + /// kept it. + pub fn set(self: *@This(), err: E) bool { + return self.code.cmpxchgStrong(0, @intFromError(err), .release, .monotonic) == null; + } + + pub fn get(self: *const @This()) ?E { + const code = self.code.load(.acquire); + return if (code == 0) null else @errorCast(@errorFromInt(code)); + } + }; +} + +test FirstError { + var first: FirstError(error{ A, B }) = .{}; + try std.testing.expectEqual(null, first.get()); + try std.testing.expect(first.set(error.B)); + try std.testing.expect(!first.set(error.A)); + try std.testing.expectEqual(error.B, first.get().?); +} + /// Packed storage of sub-byte values /// /// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index c8de1e0c..352ce826 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -4,11 +4,14 @@ zig_library( name = "vfs", srcs = [ "base.zig", - "file.zig", + "direct_io.zig", "gcs.zig", "hf.zig", "http.zig", - "parallel_read.zig", + "http_acceptance_test.zig", + "mock_server.zig", + "range_read.zig", + "request.zig", "s3.zig", ], main = "vfs.zig", diff --git a/vfs/base.zig b/vfs/base.zig index 1e66ca89..fc21c0b7 100644 --- a/vfs/base.zig +++ b/vfs/base.zig @@ -1,9 +1,50 @@ const std = @import("std"); +const builtin = @import("builtin"); const stdx = @import("stdx"); +/// `std.log.scoped`, except that test builds log `err` at `debug`: the test +/// runner counts logged errors as failures, and a test of a failure path +/// checks the error it returns. +pub fn scopedLog(comptime scope: @EnumLiteral()) type { + const scoped = std.log.scoped(scope); + return struct { + pub const debug = scoped.debug; + pub const info = scoped.info; + pub const warn = scoped.warn; + + pub fn err(comptime format: []const u8, args: anytype) void { + if (builtin.is_test) scoped.debug(format, args) else scoped.err(format, args); + } + }; +} + +/// How a backend's files are best read (`VFS.readHints`). No one source +/// request size suits every backend: warm local 8 MiB requests on one B70 +/// reached 27.05 GiB/s against 21.33 at 32 MiB, with a quarter of the pinned +/// memory; real AWS plateaued near 950 MiB/s, and 32 MiB was no faster. +pub const ReadHints = struct { + /// Local files, through a VFS or not. + pub const local: ReadHints = .{ .read_chunk_size = 8 * 1024 * 1024 }; + + /// Smallest source request expected to avoid excessive per-request cost + /// or backend rate limiting. + read_chunk_size: usize = 16 * 1024 * 1024, + /// The backend benefits from opening enough source calls to hide request + /// latency before the first response arrives. + high_latency: bool = false, +}; + +pub const Backend = struct { + io: std.Io, + read_hints: ReadHints = .{}, +}; + pub const VFSBase = struct { - inner: std.Io, + /// Aligned like the i96 durations and timestamps a backend holds (its + /// `request.Governor`), so `@fieldParentPtr("base", ...)` needs no + /// `@alignCast`. + inner: std.Io align(@alignOf(std.Io.Duration)), pub fn init(io: std.Io) VFSBase { return .{ .inner = io }; diff --git a/vfs/direct_io.zig b/vfs/direct_io.zig new file mode 100644 index 00000000..4d758589 --- /dev/null +++ b/vfs/direct_io.zig @@ -0,0 +1,366 @@ +//! Direct I/O on the descriptor of a local file. A VFS handle's `Mode` sets +//! the flag on its descriptor when a reader asks for it under a `Policy`, +//! and from then on issues that descriptor's positional reads here rather +//! than through the inner `std.Io`: the kernel answers an unsupported direct +//! read with `EINVAL` or `EFAULT`, which the threaded `std.Io` treats as a +//! programmer error, whereas here it is a refusal that clears the flag and +//! leaves the read to the VFS, buffered. Everything below `Mode` is Linux +//! only and reached from code gated on `supported`. +//! +//! Neither path wins everywhere, and residency alone cannot predict which +//! does: warm replicated Llama-3.1-8B on eight MI300X took ~1.10 s buffered +//! but ~4.53 s forced direct from a slow storage extent; on four GB300 with +//! four NVMe drives in RAID0, direct took ~0.30 s versus ~0.32 s warm +//! buffered. These are loader times, not disk-only rates. +const std = @import("std"); +const builtin = @import("builtin"); + +const linux = std.os.linux; + +const log = std.log.scoped(.@"zml/vfs"); + +pub const supported = builtin.target.os.tag == .linux and @hasField(std.posix.O, "DIRECT"); + +/// Alignment of a direct read's offset, every buffer and the total length. +/// 4 KiB covered the measured filesystems; this is not a statx(STATX_DIOALIGN) +/// query. A filesystem requiring more alignment may reject the first read, +/// after which the VFS demotes the handle to buffered. +pub const alignment: usize = 4 * 1024; + +/// `Policy.auto` reads a file directly when at most this fraction of its +/// sampled pages is cached. +const cold_fraction: f64 = 0.5; + +/// Whether a local file is read past the page cache. A direct read comes +/// from the disk even when the cache holds the data; it also leaves the cache +/// as it found it, so a file read directly is still cold for the next load +/// (the `auto` probe caches what it samples, elsewhere on every probe: +/// `cachedFraction`). +/// Raw warm-cache reads exceeded disk throughput in the measured setups, +/// but the loader can still favor direct reads: page-cache copies compete +/// with DMA for host memory bandwidth. Conversely, repeated loads from a +/// slow disk can benefit greatly from warming the cache with `off`. The +/// contrasting GB300/MI300X load results are in this file's header. +pub const Policy = enum { + off, + /// Every file the filesystem allows. + on, + /// A file mostly out of the page cache when asked, buffered for a cached + /// one. Residency is sampled: `residency_samples` pages spread over the + /// file, read with `RWF_NOWAIT`, which answers `EAGAIN` for a page that + /// is not cached, needs no permission on the file and costs the same + /// whatever its size. Unknown counts as cached. + auto, +}; + +/// A local handle's read mode: undecided until a reader asks for direct +/// reads (`VFS.useDirectIo`), then buffered or direct for the rest of the +/// handle's life, since the flag belongs to the open file description and a +/// reader that was answered buffered plans exact reads that a direct +/// descriptor would reject. A direct handle goes back to buffered once the +/// kernel rejected a read, or a read went through the inner `Io`. Each +/// method takes the inner `Io` and the handle's file on it. +pub const Mode = struct { + /// Changes only under `mutex`, together with the flag; reads only load it. + state: std.atomic.Value(State) = .init(.undecided), + mutex: std.Io.Mutex = .init, + + const State = enum(u8) { undecided, buffered, direct }; + + /// Whether `file` reads direct, deciding an undecided handle under + /// `policy` (`VFS.useDirectIo`); `off` decides nothing. + pub fn decide(self: *Mode, io: std.Io, file: std.Io.File, policy: Policy) bool { + if (comptime !supported) return false; + if (policy == .off) return false; + switch (self.state.load(.acquire)) { + .direct => return true, + .buffered => return false, + .undecided => {}, + } + const stat = file.stat(io) catch return self.settle(io, file, false); + if (!fits(stat)) return self.settle(io, file, false); + if (policy == .auto) { + const name: Name = .{ .io = io, .file = file }; + const cached = sampleResidency(io, file, stat.size) orelse { + log.debug("{f}: page cache residency unknown; reading buffered", .{name}); + return self.settle(io, file, false); + }; + if (cached > cold_fraction) { + log.debug("{f}: {d:.0}% cached; reading buffered", .{ name, cached * 100 }); + return self.settle(io, file, false); + } + log.debug("{f}: {d:.0}% cached", .{ name, cached * 100 }); + } + return self.settle(io, file, true); + } + + /// Decides an undecided handle under the mutex, setting the flag first + /// when `direct`, and returns whether the handle reads direct: a + /// decision already taken (by a racing call, or a demotion) stands. A + /// descriptor the flag cannot go on (`enable`) is read buffered. + fn settle(self: *Mode, io: std.Io, file: std.Io.File, direct: bool) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + switch (self.state.load(.acquire)) { + .direct => return true, + .buffered => return false, + .undecided => {}, + } + if (direct) { + const name: Name = .{ .io = io, .file = file }; + const err = enable(file.handle); + if (err == .SUCCESS) { + self.state.store(.direct, .release); + log.debug("{f}: reading direct", .{name}); + return true; + } + log.debug("{f}: O_DIRECT refused ({t}); reading buffered", .{ name, err }); + } + self.state.store(.buffered, .release); + return false; + } + + /// The bytes read from `offset` into `data` past the page cache, or + /// null for the caller to read them buffered: the handle is not direct, + /// or the kernel refused the read and the handle is buffered from now + /// on (`logRefusal`). A refusal demotes rather than fails: a legitimate + /// short direct read can leave an unaligned continuation, and a + /// filesystem accepting the flag does not prove it accepts these + /// buffers (including PJRT-pinned memory) at read time. + pub fn read(self: *Mode, io: std.Io, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!?usize { + if (comptime !supported) return null; + if (self.state.load(.acquire) != .direct) return null; + switch (try readPositional(file.handle, data, offset)) { + .bytes => |bytes| return bytes, + .refused => |err| { + if (self.demote(io, file)) logRefusal(.{ .io = io, .file = file }, err, data, offset); + return null; + }, + } + } + + /// Takes a direct handle back to buffered before a read the VFS does + /// not issue itself (streaming, or as the source of a file copy): the + /// inner `Io` treats the errno of a rejected direct read as a + /// programmer error. + pub fn leave(self: *Mode, io: std.Io, file: std.Io.File) void { + if (comptime !supported) return; + if (self.state.load(.acquire) != .direct or !self.demote(io, file)) return; + const name: Name = .{ .io = io, .file = file }; + log.warn("{f}: buffered read of a direct file; reading it buffered from now on", .{name}); + } + + /// Takes a direct handle back to buffered: the flag comes off the + /// descriptor before the state says buffered, under the mutex, so the + /// inner `Io` never reads a descriptor with the flag on. True for the + /// call that did it, which then says why. + fn demote(self: *Mode, io: std.Io, file: std.Io.File) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + if (self.state.load(.acquire) != .direct) return false; + disable(file.handle); + self.state.store(.buffered, .release); + return true; + } +}; + +/// A file's size, and whether `Mode.decide` would read it direct +/// (`predict`). +pub const Prediction = struct { size: u64, direct: bool }; + +/// `Mode.decide`'s answer for `file` under `policy`, deciding and logging +/// nothing: the same rule, then the flag set on `file` and left set, so +/// `file` must be a read-only descriptor of the caller's own, which the +/// caller closes. Under `auto` the sample caches what it misses, as +/// `decide`'s does: a missed sample queues the read of its page and of the +/// kernel's readahead. A decision that follows and samples those pages +/// once the reads have landed finds the file cached and reads it buffered, +/// which may happen to any file, the likelier the fewer pages it has per +/// sample (`cachedFraction`). Null when `file` cannot be stat-ed. +pub fn predict(io: std.Io, file: std.Io.File, policy: Policy) error{Canceled}!?Prediction { + if (comptime !supported) return null; + const stat = file.stat(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return null, + }; + const direct = fits(stat) and switch (policy) { + .off => false, + .on => true, + .auto => if (sampleResidency(io, file, stat.size)) |cached| cached <= cold_fraction else false, + }; + return .{ .size = stat.size, .direct = direct and enable(file.handle) == .SUCCESS }; +} + +/// A regular file at least one unit long: the only kind read direct +/// (`Mode.decide`, `predict`). +fn fits(stat: std.Io.File.Stat) bool { + return stat.kind == .file and stat.size >= alignment; +} + +/// A local file's path in a log line, resolved only when the line is +/// written. +const Name = struct { + io: std.Io, + file: std.Io.File, + + pub fn format(self: Name, writer: *std.Io.Writer) !void { + var buffer: [std.fs.max_path_bytes]u8 = undefined; + const len = self.file.realPath(self.io, &buffer) catch return writer.writeAll(""); + try writer.writeAll(buffer[0..len]); + } +}; + +/// The kernel rejected a direct read. An aligned one is the filesystem +/// refusing direct I/O it accepted the flag for, a warning; a misaligned +/// one is the reader's, either its plan or a continuation after a short +/// read (which a network filesystem may answer), an error. Either way the +/// file is read buffered from now on. +fn logRefusal(name: Name, err: linux.E, data: []const []u8, offset: u64) void { + var total: usize = 0; + var aligned = offset % alignment == 0; + for (data) |buffer| { + if (buffer.len == 0) continue; + aligned = aligned and @intFromPtr(buffer.ptr) % alignment == 0; + total += buffer.len; + } + aligned = aligned and total % alignment == 0; + if (aligned) { + log.warn("{f}: direct read refused ({t}) at offset={d} total={d}; reading the file buffered from now on", .{ name, err, offset, total }); + } else { + log.err("{f}: misaligned read of a direct file ({t}) at offset={d} first_buffer=0x{x} total={d} alignment={d}; reading the file buffered from now on", .{ + name, + err, + offset, + if (data.len != 0) @intFromPtr(data[0].ptr) else 0, + total, + alignment, + }); + } +} + +const max_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 else std.posix.IOV_MAX; +const direct_flag: usize = @as(u32, @bitCast(std.posix.O{ .DIRECT = true })); + +/// Sets the flag on a descriptor opened for reading only, since the flag +/// constrains writes too: one opened for writing is refused with `BADF`. A +/// filesystem without direct I/O refuses the flag. +pub fn enable(fd: std.posix.fd_t) linux.E { + const current = switch (getStatusFlags(fd)) { + .flags => |flags| flags, + .err => |err| return err, + }; + const status: std.posix.O = @bitCast(@as(u32, @truncate(current))); + if (status.ACCMODE != .RDONLY) return .BADF; + return setStatusFlags(fd, current | direct_flag); +} + +/// Clears the flag, so the inner `Io` may read the descriptor again. +fn disable(fd: std.posix.fd_t) void { + const current = switch (getStatusFlags(fd)) { + .flags => |flags| flags, + .err => return, + }; + _ = setStatusFlags(fd, current & ~direct_flag); +} + +/// `fcntl(F_GETFL)`, retried on `EINTR`. +fn getStatusFlags(fd: std.posix.fd_t) union(enum) { flags: usize, err: linux.E } { + while (true) { + const rc = linux.fcntl(fd, linux.F.GETFL, 0); + switch (linux.errno(rc)) { + .SUCCESS => return .{ .flags = rc }, + .INTR => continue, + else => |err| return .{ .err = err }, + } + } +} + +/// `fcntl(F_SETFL)`, retried on `EINTR`. +fn setStatusFlags(fd: std.posix.fd_t, flags: usize) linux.E { + while (true) { + switch (linux.errno(linux.fcntl(fd, linux.F.SETFL, flags))) { + .INTR => continue, + else => |err| return err, + } + } +} + +/// A fixed count bounds the probe's cost whatever the checkpoint's size and +/// suffices for the coarse half-cached decision. A whole-file `cachestat` +/// walks every cached folio, about 7 ms per cached GiB on a 4 KiB-page +/// kernel, paid on the planner thread, and refuses a file the process can +/// read but not write. +pub const residency_samples = 32; + +/// The fraction of `residency_samples` pages spread over the file that the +/// page cache holds, or null when the filesystem cannot say (no +/// `RWF_NOWAIT`). One sample per window of the file, each `shift` bytes +/// into its window (`shift` modulo the window), unknown residency selecting +/// buffered IO. A sample that misses queues the read of its page, which a +/// direct file never reads back: with the same offsets the next probe would +/// find every sample cached and read a cold file buffered, so callers draw a +/// new `shift` for every probe. A file of at most `residency_samples` pages +/// has every page sampled whatever the shift, so its first probe caches it. +/// The header page a parser has just read is sampled only when `shift` +/// lands in it, not systematically. +fn cachedFraction(fd: std.posix.fd_t, size: u64, shift: u64) ?f64 { + var buffer: [512]u8 = undefined; + const iovec: std.posix.iovec = .{ .base = &buffer, .len = buffer.len }; + var cached: usize = 0; + const window = size / residency_samples; + for (0..residency_samples) |sample| { + const offset = window * sample + shift % @max(window, 1); + const rc = linux.preadv2(fd, @ptrCast(&iovec), 1, @intCast(offset), linux.RWF.NOWAIT); + switch (linux.errno(rc)) { + .SUCCESS => cached += @intFromBool(rc != 0), + .AGAIN => {}, + else => return null, + } + } + return @as(f64, @floatFromInt(cached)) / @as(f64, @floatFromInt(residency_samples)); +} + +/// `cachedFraction` at a fresh shift, as every probe draws one. +fn sampleResidency(io: std.Io, file: std.Io.File, size: u64) ?f64 { + var shift: u64 = undefined; + io.random(std.mem.asBytes(&shift)); + return cachedFraction(file.handle, size, shift); +} + +const ReadResult = union(enum) { + bytes: usize, + /// The kernel rejected the direct read: an unaligned offset, buffer or + /// total, a filesystem that took the flag but not the read, or memory + /// the block layer cannot pin. + refused: linux.E, +}; + +/// Reads `data` at `offset` on a descriptor with the flag set, at most +/// `IOV_MAX` buffers per call (the caller loops). A read that runs past the +/// end of the file is cut there. +fn readPositional(fd: std.posix.fd_t, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!ReadResult { + var iovecs: [max_iovecs]std.posix.iovec = undefined; + var count: usize = 0; + for (data) |buffer| { + if (buffer.len == 0) continue; + if (count == iovecs.len) break; + iovecs[count] = .{ .base = buffer.ptr, .len = buffer.len }; + count += 1; + } + if (count == 0) return .{ .bytes = 0 }; + while (true) { + const rc = linux.preadv(fd, &iovecs, count, @intCast(offset)); + switch (linux.errno(rc)) { + .SUCCESS => return .{ .bytes = rc }, + .INTR => continue, + .INVAL, .FAULT, .OPNOTSUPP => |err| return .{ .refused = err }, + .IO => return error.InputOutput, + .NOMEM => return error.SystemResources, + .AGAIN => return error.WouldBlock, + .ISDIR => return error.IsDir, + .BADF => return error.NotOpenForReading, + .NXIO, .SPIPE => return error.Unseekable, + else => |err| return std.posix.unexpectedErrno(err), + } + } +} diff --git a/vfs/file.zig b/vfs/file.zig deleted file mode 100644 index 1b077c33..00000000 --- a/vfs/file.zig +++ /dev/null @@ -1,430 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const VFSBase = @import("base.zig").VFSBase; - -const log = std.log.scoped(.@"zml/vfs/file"); - -pub const DirectIoError = error{ - UnsupportedPlatform, - MisalignedBuffer, - BufferTooSmall, - UnexpectedFcntlResult, -}; - -fn canUseDirectIO() bool { - if (builtin.target.os.tag == .linux) { - return @hasField(std.posix.O, "DIRECT"); - } - return false; -} - -fn getStatusFlags(file: std.Io.File) DirectIoError!c_int { - while (true) { - const result: c_int = std.posix.system.fcntl(file.handle, std.posix.F.GETFL, @as(c_int, 0)); - switch (std.posix.errno(result)) { - .SUCCESS => return result, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - -fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { - while (true) { - switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { - .SUCCESS => return, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - -fn useDirectIO(file: std.Io.File) DirectIoError!bool { - if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); - const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - return (flags & direct_flag) != 0; - } else { - return DirectIoError.UnsupportedPlatform; - } -} - -fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { - if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); - const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - if ((flags & direct_flag) == 0) return; - - try setStatusFlags(file, flags & ~direct_flag); - } else { - return DirectIoError.UnsupportedPlatform; - } -} - -fn switchToDirectIO(file: std.Io.File) DirectIoError!void { - if (builtin.os.tag == .linux and canUseDirectIO()) { - const flags = try getStatusFlags(file); - const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - try setStatusFlags(file, flags | direct_flag); - } else { - return DirectIoError.UnsupportedPlatform; - } -} - -pub const File = struct { - pub const Config = struct { - direct_io: bool = true, - direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), - }; - - const Handle = struct { - inner_handle: std.Io.File.Handle, - inner_flags: std.Io.File.Flags, - direct_io_handle: ?std.Io.File.Handle, - - pub fn innerFile(self: *Handle) std.Io.File { - return .{ - .handle = self.inner_handle, - .flags = self.inner_flags, - }; - } - - pub fn directIoFile(self: *Handle) std.Io.File { - return .{ - .handle = self.direct_io_handle.?, - .flags = self.inner_flags, - }; - } - }; - - allocator: std.mem.Allocator, - mutex: std.Io.Mutex = .init, - handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .empty, - config: Config, - base: VFSBase, - - pub fn init(allocator: std.mem.Allocator, inner: std.Io, config: Config) File { - return .{ - .allocator = allocator, - .config = config, - .base = .init(inner), - }; - } - - pub fn deinit(self: *File) void { - self.handles.deinit(self.allocator); - self.closed_handles.deinit(self.allocator); - } - - pub fn io(self: *File) std.Io { - return .{ - .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), - }; - } - - fn openHandle(self: *File) !struct { u32, *Handle } { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (self.closed_handles.pop()) |idx| { - return .{ idx, self.handles.at(idx) }; - } - return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; - } - - fn closeHandle(self: *File, idx: u32) !void { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - try self.closed_handles.append(self.allocator, idx); - } - - fn getFileHandle(self: *File, file: std.Io.File) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - return self.handles.at(@intCast(file.handle)); - } - - fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { - const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { - else => return std.Io.File.OpenError.Unexpected, - }; - - return self.config.direct_io and flags.mode == .read_only and file_stat.size >= self.config.direct_io_alignment.toByteUnits(); - } - - fn innerFile(self: *File, file: std.Io.File, data: []const []u8, position: u64) std.Io.File { - const handle = self.getFileHandle(file); - - if (handle.direct_io_handle == null) { - return handle.innerFile(); - } - - const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); - - if (!std.mem.isAligned(@as(usize, position), alignment_bytes)) { - return handle.innerFile(); - } - - var total_size: usize = 0; - for (data) |buf| { - if (!std.mem.isAligned(@intFromPtr(buf.ptr), alignment_bytes)) { - return handle.innerFile(); - } - total_size += buf.len; - } - - if (!std.mem.isAligned(total_size, alignment_bytes)) { - return handle.innerFile(); - } - - return handle.directIoFile(); - } - - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle = self.getFileHandle(o.file); - const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); - const inner_file = self.innerFile(o.file, o.data, position); - return self.base.inner.vtable.operate(self.base.inner.userdata, .{ - .file_read_streaming = .{ - .file = inner_file, - .data = o.data, - }, - }); - }, - .file_write_streaming => |o| { - const handle = self.getFileHandle(o.file); - return self.base.inner.vtable.operate(self.base.inner.userdata, .{ - .file_write_streaming = .{ - .file = handle.innerFile(), - .header = o.header, - .data = o.data, - .splat = o.splat, - }, - }); - }, - .device_io_control, .net_receive, .net_read, .net_send, .net_write => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } - } - - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirOpenDir(self.base.inner.userdata, dir, sub_path, options); - } - - fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirStat(self.base.inner.userdata, dir); - } - - fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirStatFile(self.base.inner.userdata, dir, sub_path, options); - } - - fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); - } - - fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); - errdefer inner_file.close(self.base.inner); - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .inner_handle = inner_file.handle, - .inner_flags = inner_file.flags, - .direct_io_handle = null, - }; - return .{ .handle = @intCast(idx), .flags = inner_file.flags }; - } - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); - errdefer inner_file.close(self.base.inner); - - const use_direct_io = try self.canOpenWithDirectIO(inner_file, flags); - - var direct_io_handle: ?std.Io.File.Handle = null; - if (use_direct_io) { - const direct_io_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); - defer if (direct_io_handle == null) direct_io_file.close(self.base.inner); - - switchToDirectIO(direct_io_file) catch |err| switch (err) { - DirectIoError.UnsupportedPlatform => { - log.debug("Direct I/O is not supported on this platform", .{}); - }, - else => { - log.err("Failed to switch to Direct I/O mode: {any}", .{err}); - return std.Io.File.OpenError.Unexpected; - }, - }; - - if (useDirectIO(direct_io_file) catch false) { - direct_io_handle = direct_io_file.handle; - } - } - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .inner_handle = inner_file.handle, - .inner_flags = inner_file.flags, - .direct_io_handle = direct_io_handle, - }; - - return .{ .handle = @intCast(idx), .flags = inner_file.flags }; - } - - fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirClose(self.base.inner.userdata, dirs); - } - - fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirRead(self.base.inner.userdata, reader, entries); - } - - fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirRealPath(self.base.inner.userdata, dir, out_buffer); - } - - fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirRealPathFile(self.base.inner.userdata, dir, path_name, out_buffer); - } - - fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileStat(self.base.inner.userdata, handle.innerFile()); - } - - fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileLength(self.base.inner.userdata, handle.innerFile()); - } - - fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - for (files) |file| { - const handle = self.getFileHandle(file); - - if (handle.direct_io_handle) |_| { - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.directIoFile()}); - } - - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.innerFile()}); - - self.closeHandle(@intCast(file.handle)) catch unreachable; - } - } - - fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); - } - - fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); - } - - fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); - } - - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = self.innerFile(file, data, offset); - return self.base.inner.vtable.fileReadPositional(self.base.inner.userdata, inner_file, data, offset); - } - - fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, handle.innerFile(), relative_offset); - } - - fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, handle.innerFile(), absolute_offset); - } - - fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, handle.innerFile(), out_buffer); - } -}; diff --git a/vfs/gcs.zig b/vfs/gcs.zig index acc7f634..6717c59b 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -3,13 +3,15 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/gcs"); -const EndpointUrl = "https://storage.googleapis.com"; const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; +const TokenUrl = "https://oauth2.googleapis.com/token"; /// Check if running on Google Compute Engine / Cloud Run host infrastructure, /// because metadata server probing should only happen on GCP. @@ -51,29 +53,11 @@ const Credentials = union(enum) { }; const ServiceAccount = struct { - pub const JwtAssertion = struct { - iss: []const u8, - scope: []const u8 = "https://www.googleapis.com/auth/devstorage.read_only", - aud: []const u8 = "https://oauth2.googleapis.com/token", - exp: i64, - iat: i64, - sub: ?[]const u8 = null, - }; - type: []const u8, client_email: []const u8, private_key: []const u8, token_uri: []const u8, quota_project_id: ?[]const u8 = null, - - pub fn makeAssertion(self: *const ServiceAccount, io_: std.Io) JwtAssertion { - const now: std.Io.Timestamp = .now(io_, .real); - return .{ - .iss = self.client_email, - .iat = now.toSeconds(), - .exp = now.addDuration(.fromSeconds(3600)).toSeconds(), - }; - } }; authorized_user: AuthorizedUser, @@ -122,159 +106,36 @@ const Credentials = union(enum) { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const GCS = struct { - const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - backend: *GCS, - uri: std.Uri, - path: []const u8, - authorization: std.http.Client.Request.Headers.Value, - }; - - const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var req = client.request( - .GET, - job.batch.uri, - .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = job.batch.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }, - ) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout, .too_many_requests => .retry(), - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; - }; + const authorization_header_size = "Bearer ".len + OAuthToken.Size; const Token = struct { - header: []u8, + /// Guards the token. A refresh is a governed request that may wait + /// out a hold or retry for minutes, so it runs under this lock, not + /// under `GCS.mutex`, which every handle lookup takes. + mutex: std.Io.Mutex = .init, + header_storage: []u8, + header_len: usize = 0, expires_at: std.Io.Timestamp, + /// Refreshes that failed other than by a cancel. A reader that + /// queued behind one fails with it instead of running its own. + failures: std.atomic.Value(u64) = .init(0), fn expired(self: *const Token, io_: std.Io) bool { const now: std.Io.Timestamp = .now(io_, .real); return now.toSeconds() >= self.expires_at.toSeconds(); } + + fn header(self: *const Token) []const u8 { + return self.header_storage[0..self.header_len]; + } }; pub const Config = struct { credentials: ?Credentials = null, endpoint_uri: std.Uri, - region: []const u8 = "auto", + /// Where tokens come from; a test points them at a local server. + metadata_uri: std.Uri = std.Uri.parse(MetadataUrl) catch unreachable, + token_uri: std.Uri = std.Uri.parse(TokenUrl) catch unreachable, }; const Handle = struct { @@ -302,49 +163,33 @@ pub const GCS = struct { allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, mutex: std.Io.Mutex = .init, - client: *std.http.Client, config: Config, token: Token, - read_pool: *ParallelRead.Pool, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, - pub const InitOpts = struct { + pub const Options = struct { credentials: ?union(enum) { json: *std.Io.Reader, metadata_server: void, } = null, endpoint_url: []const u8 = "https://storage.googleapis.com", - region: []const u8 = "auto", - read_pool: parallel_read.InitOpts = .{ - .chunk_size = 16 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, + retry: request.Retry = .{}, }; pub const InitError = error{ InvalidCredentialJson, Unexpected, - } || std.mem.Allocator.Error || std.Io.ConcurrentError; + } || std.mem.Allocator.Error; - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, opts: InitOpts) InitError!GCS { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) InitError!GCS { var arena: std.heap.ArenaAllocator = .init(allocator); errdefer arena.deinit(); - - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - const config: Config = .{ - .credentials = if (opts.credentials) |creds| switch (creds) { + .credentials = if (options.credentials) |creds| switch (creds) { .json => |reader| blk: { var json_reader: std.json.Reader = .init(allocator, reader); defer json_reader.deinit(); @@ -358,12 +203,11 @@ pub const GCS = struct { }, .metadata_server => .{ .metadata_server = {} }, } else null, - .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, opts.endpoint_url)) catch return InitError.Unexpected, - .region = try arena.allocator().dupe(u8, opts.region), + .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, options.endpoint_url)) catch return InitError.Unexpected, }; const token: Token = .{ - .header = try arena.allocator().alloc(u8, "Bearer ".len + OAuthToken.Size), + .header_storage = try arena.allocator().alloc(u8, authorization_header_size), .expires_at = .zero, }; @@ -371,10 +215,10 @@ pub const GCS = struct { .allocator = allocator, .arena = arena, .base = .init(inner), - .client = http_client, .config = config, .token = token, - .read_pool = read_pool, + // GCS answers rate limiting with `503` as well as `429`. + .governor = .init(inner, http_client, "gcs", .throttle, options.retry), }; } @@ -419,129 +263,120 @@ pub const GCS = struct { return std.Io.Dir.openFile(.cwd(), io_, path, .{}) catch null; } - fn refreshMetadataServerToken(client: *std.http.Client, buffer: []u8) !?[]const u8 { - var response_writer: std.Io.Writer = .fixed(buffer); - const result = try client.fetch(.{ - .location = .{ .url = MetadataUrl }, - .method = .GET, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - }, - .extra_headers = &.{ - .{ .name = "Metadata-Flavor", .value = "Google" }, - }, - .response_writer = &response_writer, - }); - - if (result.status != .ok) { - return null; - } - - return response_writer.buffered(); + /// The metadata server's token, through the governed loop: a throttled + /// token endpoint holds the backend like any other request. + fn refreshMetadataServerToken(self: *GCS, buffer: []u8) ![]const u8 { + var token: TokenRequest = .{ + .uri = self.config.metadata_uri, + .target = "the metadata server", + .buffer = buffer, + .extra_headers = &.{.{ .name = "Metadata-Flavor", .value = "Google" }}, + }; + return request.perform([]const u8, &self.governor, token.spec(), &token, TokenRequest.attempt); } - fn refreshAuthorizedUserToken(client: *std.http.Client, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { - var response_writer: std.Io.Writer = .fixed(buffer); - const result = try client.fetch(.{ - .location = .{ .url = "https://oauth2.googleapis.com/token" }, + fn refreshAuthorizedUserToken(self: *GCS, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { + // Its own buffer: a retried attempt sends it again after the + // response was written into `buffer`. + var payload_buffer: [2 * 1024]u8 = undefined; + const payload = try std.fmt.bufPrint(&payload_buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + }); + var token: TokenRequest = .{ + .uri = self.config.token_uri, + .target = "the ADC token endpoint", + .buffer = buffer, .method = .POST, - .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), - }), - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .content_type = .{ .override = "application/x-www-form-urlencoded" }, - }, - .response_writer = &response_writer, - }); - if (result.status != .ok) { - log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); - return error.RequestFailed; - } - return response_writer.buffered(); + .payload = payload, + .content_type = .{ .override = "application/x-www-form-urlencoded" }, + }; + return request.perform([]const u8, &self.governor, token.spec(), &token, TokenRequest.attempt); } + /// One OAuth token exchange, whose body it copies into `buffer`. + const TokenRequest = struct { + uri: std.Uri, + target: []const u8, + buffer: []u8, + method: std.http.Method = .GET, + payload: ?[]u8 = null, + content_type: std.http.Client.Request.Headers.Value = .default, + extra_headers: []const std.http.Header = &.{}, + + fn spec(self: *const TokenRequest) request.RequestSpec { + return .{ .target = self.target, .key = request.authorityOf(self.uri) }; + } + + fn attempt(self: *TokenRequest, current: request.Attempt) anyerror!request.Outcome([]const u8) { + return request.exchange([]const u8, current, self.uri, .{ + .method = self.method, + .content_type = self.content_type, + .extra_headers = self.extra_headers, + .payload = self.payload, + .redirects = .follow, + // A refused exchange's body says why (`invalid_grant`, ...). + .log_refusal_body = true, + }, self, TokenRequest.consume); + } + + fn consume(self: *TokenRequest, res: *std.http.Client.Response) anyerror![]const u8 { + var writer: std.Io.Writer = .fixed(self.buffer); + _ = try res.reader(&.{}).streamRemaining(&writer); + return writer.buffered(); + } + }; + fn refreshServiceAccountToken(io_: std.Io, client: *std.http.Client, service_account: Credentials.ServiceAccount, buffer: []u8) ![]const u8 { _ = io_; _ = client; _ = service_account; _ = buffer; return error.Unimplemented; - - // const assertion = service_account.makeAssertion(io_); - - // var tmp_buffer: [8 * 1024]u8 = undefined; - - // var buffer_writer: std.Io.Writer = .fixed(buffer); - // var signing_input_buffer: [2 * 1024]u8 = undefined; - // var signing_input_writer: std.Io.Writer = .fixed(signing_input_buffer); - - // const encoded_rs256_header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; - // try signing_input_writer.print("{s}.", .{encoded_rs256_header}); - // std.base64.url_safe_no_pad.Encoder.encodeWriter( - // signing_input_writer, - // try std.fmt.bufPrint(tmp_buffer, "{f}", .{ - // std.json.fmt(assertion, .{}), - // }), - // ); - // const signing_input = signing_input_writer.buffered(); - // _ = signing_input; - - // const result = try client.fetch(.{ - // .location = .{ .url = "https://oauth2.googleapis.com/token" }, - // .method = .POST, - // .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ - // std.fmt.alt(std.Uri.Component{ .raw = service_account.client_id }, .formatQuery), - // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), - // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), - // }), - // .headers = .{ - // .accept_encoding = .{ .override = "identity" }, - // .content_type = .{ .override = "application/x-www-form-urlencoded" }, - // }, - // .response_writer = &response_writer, - // }); - // if (result.status != .ok) { - // log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); - // return error.RequestFailed; - // } } fn refreshToken(self: *GCS) !void { var buffer: [4 * 1024]u8 = undefined; const payload = switch (self.config.credentials.?) { - .authorized_user => |authorized_user| try refreshAuthorizedUserToken(self.client, authorized_user, &buffer), - .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.client, service_account, &buffer), - .metadata_server => try refreshMetadataServerToken(self.client, &buffer) orelse return error.RequestFailed, + .authorized_user => |authorized_user| try self.refreshAuthorizedUserToken(authorized_user, &buffer), + .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.governor.client, service_account, &buffer), + .metadata_server => try self.refreshMetadataServerToken(&buffer), }; const oauth_token = try std.json.parseFromSlice(OAuthToken, self.allocator, payload, .{ .allocate = .alloc_if_needed, .ignore_unknown_fields = true, }); defer oauth_token.deinit(); - self.token = .{ - .header = try std.fmt.bufPrint(self.token.header, "Bearer {s}", .{oauth_token.value.access_token}), - .expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))), - }; + const header = try std.fmt.bufPrint(self.token.header_storage, "Bearer {s}", .{oauth_token.value.access_token}); + self.token.header_len = header.len; + self.token.expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))); } - fn getOrRefreshToken(self: *GCS) !std.http.Client.Request.Headers.Value { + fn getOrRefreshToken(self: *GCS, authorization_buffer: []u8) !std.http.Client.Request.Headers.Value { if (self.config.credentials == null) { return .omit; } + const failures = self.token.failures.load(.monotonic); + try self.token.mutex.lock(self.base.inner); + defer self.token.mutex.unlock(self.base.inner); if (self.token.expired(self.base.inner)) { - try self.refreshToken(); + // The refresh this reader waited behind failed, and its reader + // logged why: repeating it, with a retry budget of its own, would + // only keep every other queued reader waiting. + if (self.token.failures.load(.monotonic) != failures) return error.RequestFailed; + self.refreshToken() catch |err| { + if (err != error.Canceled) _ = self.token.failures.fetchAdd(1, .monotonic); + return err; + }; } - return .{ .override = self.token.header }; + // A copy: a refresh by another reader rewrites the token's storage. + return .{ .override = try std.fmt.bufPrint(authorization_buffer, "{s}", .{self.token.header()}) }; } pub fn deinit(self: *GCS) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -591,6 +426,13 @@ pub const GCS = struct { }; } + pub fn backend(self: *GCS) Backend { + return .{ + .io = self.io(), + .read_hints = .{ .high_latency = true }, + }; + } + fn openHandle(self: *GCS) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -640,9 +482,9 @@ pub const GCS = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -688,11 +530,9 @@ pub const GCS = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, - else => return std.Io.File.OpenError.Unexpected, - }; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); return .{ .inode = 0, @@ -707,18 +547,19 @@ pub const GCS = struct { }; } - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + } fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, - else => return std.Io.File.OpenError.Unexpected, - }; - var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -744,7 +585,16 @@ pub const GCS = struct { } const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + const objects = self.listObjects(handle.uri) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.OutOfMemory => return error.SystemResources, + // Logged by `exchange`. + error.RequestFailed => return error.Unexpected, + else => { + log.err("listing {s} failed: {t}", .{ handle.uri, err }); + return error.Unexpected; + }, + }; self.dir_read_states.put(self.allocator, reader, .{ .index = 0, @@ -829,10 +679,7 @@ pub const GCS = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { @@ -842,7 +689,7 @@ pub const GCS = struct { if (relative_offset >= 0) { handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); } else { - const abs_offset: u64 = @intCast(-relative_offset); + const abs_offset: u64 = @abs(relative_offset); if (abs_offset > handle.pos) { handle.pos = 0; } else { @@ -864,25 +711,6 @@ pub const GCS = struct { return path.len; } - fn getTimestamp(self: *GCS, buf: []u8) ![]const u8 { - const now_ts = std.Io.Clock.now(.real, self.base.inner); - - const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); - const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; - const year_day = epoch_secs.getEpochDay().calculateYearDay(); - const month_day = year_day.calculateMonthDay(); - const day_seconds = epoch_secs.getDaySeconds(); - - return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ - year_day.year, - @backingInt(month_day.month), - month_day.day_index + 1, - day_seconds.getHoursIntoDay(), - day_seconds.getMinutesIntoHour(), - day_seconds.getSecondsIntoMinute(), - }); - } - fn pathComponents(_: *GCS, path_: []const u8) struct { []const u8, []const u8 } { const path = std.mem.trim(u8, path_, "/"); @@ -893,43 +721,16 @@ pub const GCS = struct { } } - fn gcsUri(self: *GCS, path: []const u8) std.Uri { + /// The object's URI. A VFS path is `bucket/key`; the request target + /// needs `/bucket/key`, formatted into `buffer`, which the URI borrows. + fn gcsUri(self: *GCS, path: []const u8, buffer: []u8) !std.Uri { var uri = self.config.endpoint_uri; - uri.path = .{ .raw = path }; + const absolute = if (std.mem.startsWith(u8, path, "/")) path else std.fmt.bufPrint(buffer, "/{s}", .{path}) catch return error.NameTooLong; + uri.path = .{ .raw = absolute }; uri.query = .{ .percent_encoded = "alt=media" }; return uri; } - // fn authHeaders(self: *GCS, _: std.http.Method, _: std.Uri, _: []const u8, authorization_buffer: []u8) !struct { - // authorization: ?[]const u8, - // extra_headers: [3]?std.http.Header, - // extra_len: usize, - // } { - // if (self.config.oauth_access_token == null) { - // if (self.config.googleCredentialsJsonStr) |json| { - // const oauth = try refreshAuthorizedUserTokenFromJson(self.allocator, self.base.inner, json); - // if (self.config.oauth_access_token) |old_token| self.allocator.free(old_token); - // if (self.config.quota_project_id) |old_project| self.allocator.free(old_project); - // self.config.oauth_access_token = oauth.access_token; - // self.config.quota_project_id = oauth.quota_project_id; - // } - // } - - // const token = self.config.oauth_access_token orelse return error.MissingCredentials; - // const authorization = try std.fmt.bufPrint(authorization_buffer, "Bearer {s}", .{token}); - // var extra_headers: [3]?std.http.Header = .{ null, null, null }; - // var extra_len: usize = 0; - // if (self.config.quota_project_id) |project| { - // extra_headers[0] = .{ .name = "x-goog-user-project", .value = project }; - // extra_len = 1; - // } - // return .{ - // .authorization = authorization, - // .extra_headers = extra_headers, - // .extra_len = extra_len, - // }; - // } - fn listObjectsBody(self: *GCS, bucket: []const u8, key_prefix: []const u8) ![]u8 { var query_buf: [4096]u8 = undefined; var query_writer = std.Io.Writer.fixed(&query_buf); @@ -951,35 +752,35 @@ pub const GCS = struct { uri.path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }; uri.query = .{ .percent_encoded = query_writer.buffered() }; - var req = try self.client.request(.GET, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = try self.getOrRefreshToken(), - }, - .extra_headers = if (self.quotaProjectId()) |project| - &.{.{ .name = "x-goog-user-project", .value = project }} - else - &.{}, - }); - defer req.deinit(); + var listing: Listing = .{ .gcs = self, .uri = uri }; + return request.perform([]u8, &self.governor, .{ .target = bucket, .key = request.authorityOf(uri) }, &listing, Listing.attempt); + } - try req.sendBodiless(); + /// The bucket listing. The bearer token is copied per attempt, so one + /// that expired during a hold is refreshed before the next try. + const Listing = struct { + gcs: *GCS, + uri: std.Uri, + authorization: [authorization_header_size]u8 = undefined, - var redirect_buffer: [2 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to list object {f}", .{uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; + fn attempt(self: *Listing, current: request.Attempt) anyerror!request.Outcome([]u8) { + return request.exchange([]u8, current, self.uri, .{ + .authorization = try self.gcs.getOrRefreshToken(&self.authorization), + .extra_headers = if (self.gcs.quotaProjectId()) |project| + &.{.{ .name = "x-goog-user-project", .value = project }} + else + &.{}, + }, self, Listing.consume); } - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(self.allocator, content_len) - else - try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); - } + fn consume(self: *Listing, res: *std.http.Client.Response) anyerror![]u8 { + const allocator = self.gcs.allocator; + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(allocator, content_len) + else + try res.reader(&.{}).allocRemaining(allocator, .limited(1024 * 1024)); + } + }; fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { const bucket, const key_prefix = self.pathComponents(prefix); @@ -1030,76 +831,65 @@ pub const GCS = struct { return try results.toOwnedSlice(self.allocator); } - fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; - const path = try self.resolvePath(dir, sub_path, &path_buffer); - const uri = self.gcsUri(path); - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = try self.getOrRefreshToken(), - }, - .extra_headers = &.{}, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - const res = try req.receiveHead(&redirect_buffer); - - const size = switch (res.head.status.class()) { - .success => res.head.content_length.?, - else => switch (res.head.status) { - .not_found => return error.FileNotFound, - .unauthorized, .forbidden => return error.PermissionDenied, - else => blk: { - log.err("Failed to fetch size for {f}: {s}", .{ uri, res.head.bytes }); - break :blk error.ServerError; - }, - }, - }; - return size; + fn fetchSize(self: *GCS, path: []const u8) !u64 { + var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; + const uri = try self.gcsUri(path, &uri_path_buffer); + var head: SizeRequest = .{ .gcs = self, .uri = uri }; + return request.perform(u64, &self.governor, .{ .target = path, .key = request.authorityOf(uri) }, &head, SizeRequest.attempt); } - fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { - const read_size = parallel_read.readSize(handle.size, offset, data); - if (read_size == 0) return 0; + /// The object's size: one HEAD. An absent or denied object fails the + /// request (`ExchangeOptions.open`). + const SizeRequest = struct { + gcs: *GCS, + uri: std.Uri, + authorization: [authorization_header_size]u8 = undefined, - const uri = self.gcsUri(handle.uri); - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .backend = self, - .uri = uri, - .path = handle.uri, - .authorization = try self.getOrRefreshToken(), - }; - - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, - }; + fn attempt(self: *SizeRequest, current: request.Attempt) anyerror!request.Outcome(u64) { + return request.exchange(u64, current, self.uri, .{ + .method = .HEAD, + .authorization = try self.gcs.getOrRefreshToken(&self.authorization), + .open = true, + }, self, SizeRequest.consume); } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; + fn consume(_: *SizeRequest, res: *std.http.Client.Response) anyerror!u64 { + return res.head.content_length orelse error.MissingContentLength; + } + }; - return read_size; + fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { + var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; + var bearer: BearerRequest = .{ .gcs = self, .uri = try self.gcsUri(handle.uri, &uri_path_buffer) }; + return range_read.performRangeRead(&self.governor, .{ + .request = .{ .target = handle.uri, .key = request.authorityOf(bearer.uri) }, + }, &bearer, BearerRequest.prepare, data, offset, handle.size); } + + /// The bearer token is copied per attempt, so one that expired during a + /// retry delay is refreshed before the next GET. + const BearerRequest = struct { + gcs: *GCS, + uri: std.Uri, + authorization: [authorization_header_size]u8 = undefined, + + fn prepare(self: *BearerRequest, _: request.Attempt, _: std.http.Header) anyerror!range_read.PreparedRequest { + return .{ .uri = self.uri, .authorization = try self.gcs.getOrRefreshToken(&self.authorization) }; + } + }; }; +test "GCS object URIs are rooted at the endpoint" { + var backend: GCS = undefined; + backend.config = .{ .endpoint_uri = try std.Uri.parse("https://storage.googleapis.com") }; + + var path_buffer: [64]u8 = undefined; + const uri = try backend.gcsUri("bucket/dir/object.bin", &path_buffer); + var uri_buffer: [128]u8 = undefined; + const formatted = try std.fmt.bufPrint(&uri_buffer, "{f}", .{uri}); + try std.testing.expectEqualStrings("https://storage.googleapis.com/bucket/dir/object.bin?alt=media", formatted); +} + test "GCS parses XML bucket listing objects and common prefixes" { const xml = "" ++ @@ -1136,3 +926,272 @@ test "GCS splits bucket and object prefix" { try std.testing.expectEqualStrings("gcp-public-data-landsat", bucket); try std.testing.expectEqualStrings("LC08/01", prefix); } + +/// A token endpoint at `/token` and one object at `/bucket/object`, for the +/// token tests. The nth token request gets `answers[n]`, the last repeating; +/// a 200 grants `tok-`, k counting the grants. +const TokenServer = struct { + tcp: std.Io.net.Server, + group: std.Io.Group = .init, + object: []const u8, + answers: []const std.http.Status, + /// How long each token answer takes, so that readers queue behind a + /// refresh. + delay: std.Io.Duration = .fromMilliseconds(100), + /// The body every token request must POST form-encoded, if any. + form: ?[]const u8 = null, + token_requests: std.atomic.Value(usize) = .init(0), + granted: std.atomic.Value(usize) = .init(0), + object_requests: std.atomic.Value(usize) = .init(0), + /// Object requests without the last token granted. + stale_requests: std.atomic.Value(usize) = .init(0), + /// Token requests that did not POST `form`. + bad_forms: std.atomic.Value(usize) = .init(0), + first_error: stdx.FirstError(anyerror) = .{}, + + fn init(io: std.Io, object: []const u8, answers: []const std.http.Status) !TokenServer { + const address: std.Io.net.IpAddress = .{ .ip4 = .loopback(0) }; + return .{ .tcp = try address.listen(io, .{ .reuse_address = true }), .object = object, .answers = answers }; + } + + fn deinit(self: *TokenServer, io: std.Io) void { + self.group.cancel(io); + self.tcp.deinit(io); + } + + fn run(self: *TokenServer, io: std.Io) std.Io.Cancelable!void { + var connections: std.Io.Group = .init; + defer connections.cancel(io); + while (true) { + const stream = self.tcp.accept(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + _ = self.first_error.set(err); + return; + }, + }; + connections.concurrent(io, onConnection, .{ self, stream, io }) catch |err| { + stream.close(io); + _ = self.first_error.set(err); + return; + }; + } + } + + fn onConnection(self: *TokenServer, stream: std.Io.net.Stream, io: std.Io) std.Io.Cancelable!void { + defer stream.close(io); + self.answer(stream, io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => _ = self.first_error.set(err), + }; + } + + fn answer(self: *TokenServer, stream: std.Io.net.Stream, io: std.Io) !void { + var read_buffer: [8 * 1024]u8 = undefined; + var reader = stream.reader(io, &read_buffer); + var write_buffer: [8 * 1024]u8 = undefined; + var writer = stream.writer(io, &write_buffer); + var server: std.http.Server = .init(&reader.interface, &writer.interface); + var req = try server.receiveHead(); + + if (std.mem.eql(u8, req.head.target, "/token")) { + const n = self.token_requests.fetchAdd(1, .acq_rel); + if (self.form) |form| { + const content_type = req.head.content_type orelse ""; + const form_encoded = req.head.method == .POST and std.mem.eql(u8, content_type, "application/x-www-form-urlencoded"); + var body_buffer: [512]u8 = undefined; + var body: std.Io.Writer = .fixed(&body_buffer); + _ = try req.readerExpectNone(&.{}).streamRemaining(&body); + if (!form_encoded or !std.mem.eql(u8, body.buffered(), form)) _ = self.bad_forms.fetchAdd(1, .acq_rel); + } + try io.sleep(self.delay, .awake); + const status = self.answers[@min(n, self.answers.len - 1)]; + if (status != .ok) return req.respond("{\"error\":\"invalid_grant\"}", .{ .status = status, .keep_alive = false }); + var token_buffer: [64]u8 = undefined; + const token = try std.fmt.bufPrint(&token_buffer, "{{\"access_token\":\"tok-{d}\",\"expires_in\":3600}}", .{self.granted.fetchAdd(1, .acq_rel) + 1}); + return req.respond(token, .{ .keep_alive = false }); + } + + if (!std.mem.eql(u8, req.head.target, "/bucket/object?alt=media")) return error.UnexpectedTarget; + _ = self.object_requests.fetchAdd(1, .acq_rel); + var authorization: []const u8 = ""; + var range: ?[]const u8 = null; + var headers = req.iterateHeaders(); + while (headers.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "authorization")) authorization = header.value; + if (std.ascii.eqlIgnoreCase(header.name, "range")) range = header.value; + } + var expected_buffer: [32]u8 = undefined; + const expected = try std.fmt.bufPrint(&expected_buffer, "Bearer tok-{d}", .{self.granted.load(.acquire)}); + if (!std.mem.eql(u8, authorization, expected)) _ = self.stale_requests.fetchAdd(1, .acq_rel); + if (req.head.method == .HEAD) return req.respond(self.object, .{ .keep_alive = false }); + + const spec = (range orelse return error.MissingRange)["bytes=".len..]; + const dash = std.mem.indexOfScalar(u8, spec, '-') orelse return error.InvalidRange; + const first = try std.fmt.parseInt(usize, spec[0..dash], 10); + const last = try std.fmt.parseInt(usize, spec[dash + 1 ..], 10); + var content_range_buffer: [64]u8 = undefined; + return req.respond(self.object[first .. last + 1], .{ + .status = .partial_content, + .keep_alive = false, + .extra_headers = &.{.{ + .name = "Content-Range", + .value = try std.fmt.bufPrint(&content_range_buffer, "bytes {d}-{d}/{d}", .{ first, last, self.object.len }), + }}, + }); + } +}; + +/// A GCS whose tokens and object come from a `TokenServer`. It must not +/// move once set up. +const TokenFixture = struct { + object: [4096]u8 = undefined, + server: TokenServer = undefined, + client: std.http.Client = undefined, + gcs: GCS = undefined, + endpoint_buffer: [32]u8 = undefined, + token_url_buffer: [48]u8 = undefined, + + fn setUp( + self: *TokenFixture, + credentials: @FieldType(GCS.Options, "credentials"), + answers: []const std.http.Status, + max_retries: usize, + ) !void { + const io = std.testing.io; + for (&self.object, 0..) |*byte, i| byte.* = @truncate(i *% 7 +% 3); + self.server = try .init(io, &self.object, answers); + errdefer self.server.deinit(io); + try self.server.group.concurrent(io, TokenServer.run, .{ &self.server, io }); + self.client = .{ .allocator = std.testing.allocator, .io = io }; + errdefer self.client.deinit(); + const port = self.server.tcp.socket.address.getPort(); + const token_uri: std.Uri = try .parse(try std.fmt.bufPrint(&self.token_url_buffer, "http://127.0.0.1:{d}/token", .{port})); + self.gcs = try .init(std.testing.allocator, io, &self.client, .{ + .credentials = credentials, + .endpoint_url = try std.fmt.bufPrint(&self.endpoint_buffer, "http://127.0.0.1:{d}", .{port}), + .retry = .{ + .max_retries = max_retries, + .initial_delay = .fromMilliseconds(5), + .max_delay = .fromMilliseconds(5), + .max_hold = .fromMilliseconds(20), + }, + }); + self.gcs.config.metadata_uri = token_uri; + self.gcs.config.token_uri = token_uri; + } + + fn tearDown(self: *TokenFixture) void { + self.gcs.deinit(); + self.client.deinit(); + self.server.deinit(std.testing.io); + } + + /// Stops the server and returns the first error it met. + fn stop(self: *TokenFixture) !void { + self.server.group.cancel(std.testing.io); + if (self.server.first_error.get()) |err| return err; + } + + /// `readers` concurrent reads of 256 bytes each, started together; + /// returns how many failed. Every read that succeeds must match. + fn readConcurrently(self: *TokenFixture, file: std.Io.File, comptime readers: usize) !usize { + const io = std.testing.io; + const Reader = struct { + fn run(gcs_io: std.Io, file_: std.Io.File, output: []u8, offset: u64, go: *std.Io.Event, result: *anyerror!usize) void { + go.wait(gcs_io) catch |err| { + result.* = err; + return; + }; + result.* = file_.readPositional(gcs_io, &.{output}, offset); + } + }; + var outputs: [readers][256]u8 = undefined; + var results: [readers]anyerror!usize = undefined; + var go: std.Io.Event = .unset; + var group: std.Io.Group = .init; + defer group.cancel(io); + for (&outputs, &results, 0..) |*output, *result, i| { + try group.concurrent(io, Reader.run, .{ self.gcs.io(), file, output, i * output.len, &go, result }); + } + go.set(io); + try group.await(io); + + var failed: usize = 0; + for (&outputs, results, 0..) |*output, result, i| { + const n = result catch { + failed += 1; + continue; + }; + try std.testing.expectEqual(output.len, n); + try std.testing.expectEqualSlices(u8, self.object[i * output.len ..][0..output.len], output); + } + return failed; + } +}; + +test "GCS refreshes an expired token once for concurrent readers, and each sends the new one" { + var fixture: TokenFixture = .{}; + try fixture.setUp(.{ .metadata_server = {} }, &.{.ok}, 0); + defer fixture.tearDown(); + const server = &fixture.server; + const gcs_io = fixture.gcs.io(); + const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); + defer file.close(gcs_io); + try std.testing.expectEqual(1, server.token_requests.load(.acquire)); + + fixture.gcs.token.expires_at = .zero; + try std.testing.expectEqual(0, try fixture.readConcurrently(file, 16)); + try fixture.stop(); + try std.testing.expectEqual(2, server.token_requests.load(.acquire)); + try std.testing.expectEqual(17, server.object_requests.load(.acquire)); + try std.testing.expectEqual(0, server.stale_requests.load(.acquire)); +} + +test "GCS fails the readers queued behind a failed token refresh without repeating it" { + const cases = [_]struct { status: std.http.Status, max_retries: usize, token_requests: usize }{ + // The open's request, then one refused refresh. + .{ .status = .unauthorized, .max_retries = 0, .token_requests = 2 }, + // The open's request, then one refresh and its 2 retries. + .{ .status = .internal_server_error, .max_retries = 2, .token_requests = 4 }, + }; + for (cases) |case| { + var fixture: TokenFixture = .{}; + try fixture.setUp(.{ .metadata_server = {} }, &.{ .ok, case.status }, case.max_retries); + defer fixture.tearDown(); + const server = &fixture.server; + const gcs_io = fixture.gcs.io(); + const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); + defer file.close(gcs_io); + + fixture.gcs.token.expires_at = .zero; + try std.testing.expectEqual(16, try fixture.readConcurrently(file, 16)); + try fixture.stop(); + try std.testing.expectEqual(case.token_requests, server.token_requests.load(.acquire)); + try std.testing.expectEqual(1, server.object_requests.load(.acquire)); + } +} + +test "GCS resends the ADC refresh form on a retry, and a throttled token endpoint holds the backend" { + var json: std.Io.Reader = .fixed( + \\{"type":"authorized_user","client_id":"cid.apps.googleusercontent.com","client_secret":"GOCSPX-sec","refresh_token":"1//0abc-def_ghi","account":"","universe_domain":"googleapis.com"} + ); + var fixture: TokenFixture = .{}; + try fixture.setUp(.{ .json = &json }, &.{ .too_many_requests, .internal_server_error, .ok }, 2); + defer fixture.tearDown(); + const server = &fixture.server; + server.delay = .zero; + server.form = "grant_type=refresh_token&client_id=cid.apps.googleusercontent.com&client_secret=GOCSPX-sec&refresh_token=1//0abc-def_ghi"; + const gcs_io = fixture.gcs.io(); + const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); + defer file.close(gcs_io); + + try std.testing.expectEqual(0, try fixture.readConcurrently(file, 4)); + try fixture.stop(); + const stats = &fixture.gcs.governor.stats; + try std.testing.expectEqual(1, stats.holds.load(.acquire)); + try std.testing.expectEqual(1, stats.retries.load(.acquire)); + try std.testing.expectEqual(3, server.token_requests.load(.acquire)); + try std.testing.expectEqual(0, server.bad_forms.load(.acquire)); + try std.testing.expectEqual(0, server.stale_requests.load(.acquire)); +} diff --git a/vfs/hf.zig b/vfs/hf.zig index 8770fe18..2989d61f 100644 --- a/vfs/hf.zig +++ b/vfs/hf.zig @@ -2,160 +2,13 @@ const std = @import("std"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/hf"); -const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - uri: std.Uri, - url: []const u8, - authorization: std.http.Client.Request.Headers.Value, - }; - - pub const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var req = client.request(.GET, job.batch.uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = job.batch.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout => .retry(), - .too_many_requests => b: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { - var parts = std.mem.splitScalar(u8, header.value, ';'); - while (parts.next()) |part| { - const p = std.mem.trim(u8, part, " \t"); - if (std.mem.startsWith(u8, p, "t=")) { - const seconds = std.fmt.parseInt(u32, p[2..], 10) catch continue; - break :b .{ .retry_after = .fromSeconds(seconds) }; - } - } - } - } - break :b .retry(); - }, - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; -}; - pub const API = struct { const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}?expand=false&recursive=true&limit=1000"; const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; @@ -200,16 +53,10 @@ const RepoKey = struct { }; pub const HF = struct { - pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{ - // Chunk size needs to be big enough to avoid hitting the rate limits of HF. - .chunk_size = 32 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, + pub const Options = struct { + /// Sent as a bearer token to the Hub itself, never to its CDN. + token: ?[]const u8 = null, + retry: request.Retry = .{}, }; pub const Repo = struct { @@ -243,13 +90,17 @@ pub const HF = struct { type: Type, uri: []const u8, + download_url: ?[]const u8, pos: u64, size: u64, - pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, size: u64) !Handle { + pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, download_url: ?[]const u8, size: u64) !Handle { + const owned_uri = try allocator.dupe(u8, uri); + errdefer allocator.free(owned_uri); return .{ .type = handle_type, - .uri = try allocator.dupe(u8, uri), + .uri = owned_uri, + .download_url = if (download_url) |url| try allocator.dupe(u8, url) else null, .pos = 0, .size = size, }; @@ -257,39 +108,37 @@ pub const HF = struct { pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { allocator.free(self.uri); + if (self.download_url) |url| allocator.free(url); } }; allocator: std.mem.Allocator, + /// Guards the handle table and the handles' download URLs. mutex: std.Io.Mutex = .init, - client: *std.http.Client, authorization: std.http.Client.Request.Headers.Value, - read_pool: *ParallelRead.Pool, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, + /// Guards `trees`. A tree fetch is a governed request that may wait out + /// a hold or retry for minutes, so it runs under this lock, not under + /// `mutex`, which every read takes. + trees_mutex: std.Io.Mutex = .init, trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8, opts: InitOpts) !HF { - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !HF { var self: HF = .{ .allocator = allocator, .base = .init(inner), - .client = http_client, - .authorization = if (hf_token) |token| blk: { + .authorization = if (options.token) |token| blk: { break :blk .{ .override = try std.fmt.allocPrint(allocator, "Bearer {s}", .{std.mem.trim(u8, token, " \t\n\r")}), }; } else blk: { break :blk .default; }, - .read_pool = read_pool, + .governor = .init(inner, http_client, "hf", .server_failure, options.retry), }; errdefer switch (self.authorization) { .default, .omit => {}, @@ -321,13 +170,11 @@ pub const HF = struct { if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); - return init(allocator, inner, http_client, hf_token, .{}); + return init(allocator, inner, http_client, .{ .token = hf_token }); } pub fn deinit(self: *HF) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -383,6 +230,16 @@ pub const HF = struct { }; } + pub fn backend(self: *HF) Backend { + return .{ + .io = self.io(), + .read_hints = .{ + .read_chunk_size = 32 * 1024 * 1024, + .high_latency = true, + }, + }; + } + fn openHandle(self: *HF) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -429,39 +286,36 @@ pub const HF = struct { return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); } - fn getOrFetchTree(self: *HF, repo: Repo) !*std.ArrayList(TreeNode) { + /// The repository's listing, fetched once. The lock is held across the + /// fetch, so every open, stat or listing of this backend waits for it, + /// cached or not. The map is read and written under `trees_mutex`, since + /// an insert may move its entries; the nodes themselves never move once + /// fetched, so the returned slice stays valid without the lock. + fn getOrFetchTree(self: *HF, repo: Repo) ![]const TreeNode { const key = repo.toKey(); var cache_key_buffer: [512]u8 = undefined; - const cache_key_lookup = try key.format(&cache_key_buffer); - if (self.trees.getPtr(cache_key_lookup)) |tree_ptr| return tree_ptr; + try self.trees_mutex.lock(self.base.inner); + defer self.trees_mutex.unlock(self.base.inner); + if (self.trees.get(cache_key_lookup)) |tree| return tree.items; + + // Room first: once fetched, the listing cannot fail to be kept. + try self.trees.ensureUnusedCapacity(self.allocator, 1); const cache_key = try key.allocFormat(self.allocator); errdefer self.allocator.free(cache_key); - - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - const gop = try self.trees.getOrPut(self.allocator, cache_key); - errdefer _ = self.trees.remove(cache_key); - - if (gop.found_existing) { - self.allocator.free(cache_key); - return gop.value_ptr; - } - - const tree_root = try self.fetchTreeFromAPI(repo); - gop.value_ptr.* = tree_root; - - return gop.value_ptr; + const tree = try self.fetchTreeFromAPI(repo); + self.trees.putAssumeCapacityNoClobber(cache_key, tree); + return tree.items; } + /// The repository listing, through the governed loop like every other + /// request. It runs under `trees_mutex` from `getOrFetchTree`, never + /// under `mutex`, so reads of open files go on while it waits. fn fetchTreeFromAPI(self: *HF, repo: Repo) !std.ArrayList(TreeNode) { var url_buffer: [8 * 1024]u8 = undefined; - var redirect_buffer: [8 * 1024]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, API.TREE_URL_TEMPLATE, .{ .repo = repo.repo, .model = repo.model, @@ -471,40 +325,64 @@ pub const HF = struct { const uri = try std.Uri.parse(url); log.info("Fetching from HF: {s}", .{url}); - var req = try self.client.request(.GET, uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = self.authorization, - }, - }); - defer req.deinit(); + var fetch: TreeFetch = .{ .hf = self, .uri = uri }; + return request.perform(std.ArrayList(TreeNode), &self.governor, .{ .target = url, .key = request.authorityOf(uri) }, &fetch, TreeFetch.attempt); + } - try req.sendBodiless(); + const TreeFetch = struct { + hf: *HF, + uri: std.Uri, - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to fetch tree: status={}", .{res.head.status}); - return error.RequestFailed; + fn attempt(self: *TreeFetch, current: request.Attempt) anyerror!request.Outcome(std.ArrayList(TreeNode)) { + return request.exchange(std.ArrayList(TreeNode), current, self.uri, .{ + .authorization = self.hf.authorization, + .open = true, + .redirects = .follow, + }, self, TreeFetch.consume); } - const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length.?); - defer self.allocator.free(body); + fn consume(self: *TreeFetch, res: *std.http.Client.Response) anyerror!std.ArrayList(TreeNode) { + const allocator = self.hf.allocator; + const body = try res.reader(&.{}).readAlloc(allocator, res.head.content_length orelse return error.MissingContentLength); + defer allocator.free(body); - const parsed = try std.json.parseFromSlice( - []API.Tree, - self.allocator, - body, - .{ .ignore_unknown_fields = true }, - ); - defer parsed.deinit(); + const parsed = try std.json.parseFromSlice( + []API.Tree, + allocator, + body, + .{ .ignore_unknown_fields = true }, + ); + defer parsed.deinit(); - var tree_root: std.ArrayList(TreeNode) = .empty; - for (parsed.value) |item| { - try insertTreeNode(self.allocator, &tree_root, item); + var tree_root: std.ArrayList(TreeNode) = .empty; + errdefer { + for (tree_root.items) |*node| node.deinit(allocator); + tree_root.deinit(allocator); + } + for (parsed.value) |item| { + try insertTreeNode(allocator, &tree_root, item); + } + return tree_root; } + }; - return tree_root; + /// The URL the file's bytes are read from, copied into `out_buffer`: + /// where the Hub's redirect chain ends, a signed CDN URL for an LFS file. + fn resolveDownloadUrl(self: *HF, repo: Repo, out_buffer: []u8) ![]const u8 { + var url_buffer: [8 * 1024]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); + // Logs name the file: past the first hop the URL is a signed one, + // which is a credential. + var target_buffer: [1024]u8 = undefined; + const target = std.fmt.bufPrint(&target_buffer, "{s}/{s}@{s}/{s}", .{ repo.repo, repo.model, repo.rev, repo.path }) catch repo.path; + var chain: request.HeadChain = .{}; + const end = try chain.resolve(&self.governor, target, url, 8, self, hubAuthorization); + return std.fmt.bufPrint(out_buffer, "{s}", .{end.url}); + } + + /// Credentials only travel to the Hub itself, never to its CDN. + fn hubAuthorization(self: *HF, url: []const u8) std.http.Client.Request.Headers.Value { + return if (std.mem.startsWith(u8, url, "https://huggingface.co/")) self.authorization else .omit; } fn findDirChildren(tree_items: []const TreeNode, dir_path: []const u8) ?[]const TreeNode { @@ -570,16 +448,16 @@ pub const HF = struct { return null; } - fn getSizeFromTree(tree: *std.ArrayList(TreeNode), path: []const u8) ?u64 { + fn getSizeFromTree(tree: []const TreeNode, path: []const u8) ?u64 { if (path.len == 0) { var total: u64 = 0; - for (tree.items) |*node| { + for (tree) |*node| { if (node.kind == .file) total += node.size; } return total; } - const node = findNode(tree.items, path) orelse return null; + const node = findNode(tree, path) orelse return null; return node.size; } @@ -640,9 +518,9 @@ pub const HF = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -665,11 +543,11 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.OpenError.BadPathName; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.OpenError.Unexpected; + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.OpenError.FileNotFound; const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .directory, full_path, size) catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, full_path, null, size) catch return std.Io.Dir.OpenError.Unexpected; return .{ .handle = @intCast(idx) }; } @@ -698,16 +576,16 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.StatFileError.Unexpected; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.StatFileError.Unexpected; - const node = findNode(tree.items, repo.path) orelse return std.Io.Dir.StatFileError.IsDir; - + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); + // The repository itself is a directory. + if (repo.path.len == 0) return std.Io.Dir.StatFileError.IsDir; + const node = findNode(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; if (node.kind == .directory) return std.Io.Dir.StatFileError.IsDir; - const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; return .{ .inode = @intCast(0), .nlink = 0, - .size = size, + .size = node.size, .permissions = .fromMode(0o444), .kind = .file, .atime = null, @@ -724,11 +602,11 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.AccessError.FileNotFound; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.AccessError.Unexpected; + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); if (repo.path.len == 0) return; - const node = findNode(tree.items, repo.path); + const node = findNode(tree, repo.path); if (node == null) return std.Io.Dir.AccessError.FileNotFound; } @@ -739,11 +617,17 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.File.OpenError.BadPathName; - const tree = self.getOrFetchTree(repo) catch return std.Io.File.OpenError.Unexpected; - const size = getSizeFromTree(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); + // A directory, the repository itself included, has no download URL. + if (repo.path.len == 0) return std.Io.File.OpenError.IsDir; + const node = findNode(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; + if (node.kind == .directory) return std.Io.File.OpenError.IsDir; + var download_url_buffer: [16 * 1024]u8 = undefined; + // Logged by the file's path: its signed URL would be a credential. + const download_url = self.resolveDownloadUrl(repo, &download_url_buffer) catch |err| return self.governor.openFailure(full_path, err); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .file, full_path, size) catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, full_path, download_url, node.size) catch return std.Io.File.OpenError.Unexpected; return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; } @@ -765,9 +649,12 @@ pub const HF = struct { const handle = self.getDirHandle(reader.dir); const repo = Repo.parse(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.Reader.Error.Unexpected; + const tree = self.getOrFetchTree(repo) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return std.Io.Dir.Reader.Error.Unexpected, + }; - const children = findDirChildren(tree.items, repo.path) orelse &.{}; + const children = findDirChildren(tree, repo.path) orelse &.{}; self.dir_read_states.put(self.allocator, reader, .{ .children = children, @@ -845,20 +732,18 @@ pub const HF = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); + // Saturates at both ends; `@abs` takes `minInt(i64)` too. handle.pos = if (relative_offset >= 0) - handle.pos + @as(u64, @intCast(relative_offset)) + handle.pos +| @abs(relative_offset) else - handle.pos - @as(u64, @intCast(-relative_offset)); + handle.pos -| @abs(relative_offset); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { @@ -877,44 +762,125 @@ pub const HF = struct { fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { if (offset >= handle.size) return 0; - var read_size: usize = 0; - for (data) |buf| read_size += buf.len; - read_size = @intCast(@min(handle.size - offset, read_size)); - if (read_size == 0) return 0; - - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - const repo: Repo = try .parse(handle.uri); - var url_buffer: [8 * 1024]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); - const uri: std.Uri = try .parse(url); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .uri = uri, - .url = url, - .authorization = self.authorization, + // Another read of the file may replace the handle's URL + // (`DataRequest.reresolve`): each read sends its own copy. + const url = url: { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + break :url try self.allocator.dupe(u8, handle.download_url orelse return error.InvalidHandle); }; + defer self.allocator.free(url); + var download: DataRequest = .{ .hf = self, .handle = handle, .uri = try .parse(url), .url = url }; + defer if (download.re_resolved) |owned| self.allocator.free(owned); + return range_read.performRangeRead(&self.governor, .{ + .request = .{ + // The file, not its signed URL: that is a credential, and + // a re-resolve replaces it. + .target = handle.uri, + .key = request.authorityOf(download.uri), + // A signed CDN URL can expire while the backend is held; + // the CDN then answers 401 or 403, which is not retryable in + // general but is here, once, after re-resolving it. + .retry_once = &.{ .unauthorized, .forbidden }, + }, + }, &download, DataRequest.prepare, data, offset, handle.size); + } - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, + const DataRequest = struct { + hf: *HF, + handle: *Handle, + /// Parsed from `url`, whose bytes it slices. + uri: std.Uri, + /// The read's own copy of the URL it sends. + url: []const u8, + /// The URL `reresolve` gave this read, owned by the read; at most + /// one per read. + re_resolved: ?[]const u8 = null, + + fn prepare(self: *DataRequest, attempt: request.Attempt, _: std.http.Header) anyerror!range_read.PreparedRequest { + if (attempt.previous_status) |status| switch (status) { + .unauthorized, .forbidden => { + if (self.re_resolved != null) return error.PermissionDenied; + try self.reresolve(); + }, + else => {}, + }; + return .{ + .uri = self.uri, + .authorization = self.hf.hubAuthorization(self.url), }; } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; - - return read_size; - } + /// Resolves the download URL again through the governed HEAD chain + /// and stores a copy on the handle, so later reads of the same file + /// use the fresh one too. Reads copy the handle's URL under + /// `hf.mutex`, so none still uses the one this replaces. + fn reresolve(self: *DataRequest) !void { + const allocator = self.hf.allocator; + const repo = try Repo.parse(self.handle.uri); + var url_buffer: [16 * 1024]u8 = undefined; + const resolved = try self.hf.resolveDownloadUrl(repo, &url_buffer); + // Everything that can fail comes before anything is handed over. + const owned = try allocator.dupe(u8, resolved); + errdefer allocator.free(owned); + const uri: std.Uri = try .parse(owned); + const for_handle = try allocator.dupe(u8, resolved); + { + self.hf.mutex.lockUncancelable(self.hf.base.inner); + defer self.hf.mutex.unlock(self.hf.base.inner); + if (self.handle.download_url) |previous| allocator.free(previous); + self.handle.download_url = for_handle; + } + self.re_resolved = owned; + self.url = owned; + self.uri = uri; + log.warn("hf: re-resolved the download URL of {s}", .{self.handle.uri}); + } + }; }; + +test "HF reports a path absent from the repository listing as FileNotFound" { + const allocator = std.testing.allocator; + var client: std.http.Client = .{ .allocator = allocator, .io = std.testing.io }; + defer client.deinit(); + var hf: HF = try .init(allocator, std.testing.io, &client, .{}); + defer hf.deinit(); + + // The listing `getOrFetchTree` would keep, so no request is sent. + var tree: std.ArrayList(TreeNode) = .empty; + try HF.insertTreeNode(allocator, &tree, .{ .oid = "", .path = "config.json", .type = "file", .size = 2 }); + try HF.insertTreeNode(allocator, &tree, .{ .oid = "", .path = "nested/model.safetensors", .type = "file", .size = 8 }); + try hf.trees.put(allocator, try allocator.dupe(u8, "owner/model@main"), tree); + + const io = hf.io(); + try std.testing.expectEqual(@as(u64, 2), (try std.Io.Dir.statFile(.cwd(), io, "owner/model/config.json", .{})).size); + try std.Io.Dir.access(.cwd(), io, "owner/model/nested/model.safetensors", .{}); + try std.testing.expectError(error.IsDir, std.Io.Dir.statFile(.cwd(), io, "owner/model/nested", .{})); + try std.testing.expectError(error.IsDir, std.Io.Dir.statFile(.cwd(), io, "owner/model", .{})); + // Opening a directory as a file sends no request. + try std.testing.expectError(error.IsDir, std.Io.Dir.openFile(.cwd(), io, "owner/model/nested", .{})); + try std.testing.expectError(error.IsDir, std.Io.Dir.openFile(.cwd(), io, "owner/model", .{})); + + try std.testing.expectError(error.FileNotFound, std.Io.Dir.statFile(.cwd(), io, "owner/model/missing.json", .{})); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.access(.cwd(), io, "owner/model/missing.json", .{})); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.openFile(.cwd(), io, "owner/model/missing.json", .{})); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.openDir(.cwd(), io, "owner/model/missing", .{})); +} + +test "HF sends its token to the Hub only" { + const allocator = std.testing.allocator; + var client: std.http.Client = .{ .allocator = allocator, .io = std.testing.io }; + defer client.deinit(); + var hf: HF = try .init(allocator, std.testing.io, &client, .{ .token = "hf_token" }); + defer hf.deinit(); + + try std.testing.expectEqualStrings("Bearer hf_token", hf.hubAuthorization("https://huggingface.co/owner/model/resolve/main/model.safetensors").override); + // Not the Hub: the CDNs its resolve URLs redirect to, a look-alike + // host, plain http. + for ([_][]const u8{ + "https://cas-bridge.xethub.hf.co/xet-bridge-us/0123?X-Amz-Signature=abc", + "https://cdn-lfs.huggingface.co/repos/00/11/0123", + "https://huggingface.co.evil.com/owner/model/resolve/main/model.safetensors", + "http://huggingface.co/owner/model/resolve/main/model.safetensors", + }) |url| try std.testing.expect(hf.hubAuthorization(url) == .omit); +} diff --git a/vfs/http.zig b/vfs/http.zig index 9054983e..a9765d0f 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -3,10 +3,18 @@ const std = @import("std"); const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const log = std.log.scoped(.@"zml/vfs/http"); pub const HTTP = struct { + pub const Options = struct { + protocol: Protocol, + retry: request.Retry = .{}, + }; + const Handle = struct { pub const Type = enum { file, @@ -39,22 +47,23 @@ pub const HTTP = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, - client: *std.http.Client, protocol: Protocol, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, protocol: Protocol) !HTTP { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !HTTP { return .{ .allocator = allocator, .base = .init(inner), - .client = http_client, - .protocol = protocol, + .protocol = options.protocol, + .governor = .init(inner, http_client, "http", .server_failure, options.retry), }; } pub fn deinit(self: *HTTP) void { + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -94,6 +103,13 @@ pub const HTTP = struct { }; } + pub fn backend(self: *HTTP) Backend { + return .{ + .io = self.io(), + .read_hints = .{ .high_latency = true }, + }; + } + fn openHandle(self: *HTTP) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -144,9 +160,9 @@ pub const HTTP = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -193,7 +209,9 @@ pub const HTTP = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); return .{ .inode = @intCast(0), @@ -208,15 +226,19 @@ pub const HTTP = struct { }; } - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + } fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; - var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -282,20 +304,18 @@ pub const HTTP = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); + // Saturates at both ends; `@abs` takes `minInt(i64)` too. handle.pos = if (relative_offset >= 0) - handle.pos + @as(u64, @intCast(relative_offset)) + handle.pos +| @abs(relative_offset) else - handle.pos - @as(u64, @intCast(-relative_offset)); + handle.pos -| @abs(relative_offset); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { @@ -311,117 +331,31 @@ pub const HTTP = struct { return path.len; } - fn fetchSize(self: *HTTP, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; - var redirect_buffer: [8 * 1024]u8 = undefined; - var aux_buffer: []u8 = &redirect_buffer; + /// The size of the object at `url` (a resolved path without its scheme), + /// from a HEAD of the end of its redirect chain, as far as a read's GET + /// follows it (`request.max_redirects`). + fn fetchSize(self: *HTTP, url: []const u8) !u64 { + var url_buffer: [8 * 1024]u8 = undefined; + const full_url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), url }); + var chain: request.HeadChain = .{}; + const end = try chain.resolve(&self.governor, url, full_url, request.max_redirects, {}, urlAuthorization); + return end.size orelse error.MissingContentLength; + } - const scheme = @tagName(self.protocol); - const url = try self.resolvePath(dir, sub_path, &path_buffer); - const full_url = try std.fmt.bufPrint(aux_buffer, "{s}://{s}", .{ scheme, url }); - - var uri = std.Uri.parse(full_url) catch return std.Io.File.OpenError.BadPathName; - while (true) { - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var res = try req.receiveHead(&redirect_buffer); - - switch (res.head.status.class()) { - .server_error, .client_error => { - log.err("Failed to fetch tree size for {s}", .{url}); - log.err("{s}", .{res.head.bytes}); - return error.ServerError; - }, - .informational => return error.UnexpectedStatus, - .success => return res.head.content_length.?, - .redirect => { - const location = res.head.location.?; - @memcpy(aux_buffer[0..location.len], location); - uri = uri.resolveInPlace(location.len, &aux_buffer) catch unreachable; - continue; - }, - } - } + /// Only what the URL carries: std's basic authorization from its + /// userinfo, as on every GET. + fn urlAuthorization(_: void, _: []const u8) std.http.Client.Request.Headers.Value { + return .default; } fn performRead(self: *HTTP, handle: *Handle, data: []const []u8, offset: u64) !usize { - if (offset >= handle.size) return 0; - - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += @as(u64, buf.len); - } - const remaining = handle.size - offset; - const take = @min(remaining, total_bytes); - const end = offset + take - 1; - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; - }; - var url_buffer: [8 * 1024]u8 = undefined; const url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), handle.uri }); const uri: std.Uri = try .parse(url); - - var req = try self.client.request(.GET, uri, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to perform read for {s}", .{handle.uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - - if (content_range) |cr| { - if (cr.start < offset) { - try reader.discardAll(offset - cr.start); - } - } - - return try reader.readSliceShort(data[0]); - } - - const ContentRange = struct { - start: u64, - end: u64, - total: u64, - }; - - fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; + return range_read.performRangeRead(&self.governor, .{ + .request = .{ .target = url, .key = request.authorityOf(uri) }, + // The object `fetchSize` found at the end of the chain. + .redirects = .follow, + }, &uri, range_read.static, data, offset, handle.size); } }; diff --git a/vfs/http_acceptance_test.zig b/vfs/http_acceptance_test.zig new file mode 100644 index 00000000..a09464a5 --- /dev/null +++ b/vfs/http_acceptance_test.zig @@ -0,0 +1,704 @@ +const std = @import("std"); + +const GCS = @import("gcs.zig").GCS; +const HTTP = @import("http.zig").HTTP; +const MockServer = @import("mock_server.zig").MockServer; +const range_read = @import("range_read.zig"); +const vfs_request = @import("request.zig"); +const S3 = @import("s3.zig").S3; + +/// A retry configuration that never sleeps, for the tests that count +/// attempts rather than time. +fn instantRetries(max_retries: usize) vfs_request.Retry { + return .{ + .max_retries = max_retries, + .initial_delay = .fromNanoseconds(0), + .max_delay = .fromNanoseconds(0), + .max_hold = .fromNanoseconds(0), + .throttle_budget = .fromSeconds(300), + }; +} + +/// A retry configuration whose retries and holds are short enough for a test. +fn fastRetry(initial_ms: i64, budget_ms: i64) vfs_request.Retry { + return .{ + .max_retries = 5, + .initial_delay = .fromMilliseconds(initial_ms), + .max_delay = .fromMilliseconds(initial_ms), + .max_hold = .fromSeconds(30), + .throttle_budget = .fromMilliseconds(budget_ms), + }; +} + +/// A `MockServer` serving `object`, an HTTP backend with a client of its +/// own, and `/object` open through that backend. Initialized in place: the +/// backend points at the client and the server must not move once started. +const Fixture = struct { + server: MockServer, + client: std.http.Client, + http: HTTP, + file: std.Io.File, + + fn init(self: *Fixture, object: []const u8, server_opts: MockServer.Options, retry: vfs_request.Retry) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + self.server = try .init(io, object, server_opts); + errdefer self.server.deinit(io); + try self.server.start(io); + self.client = .{ .allocator = allocator, .io = io }; + errdefer self.client.deinit(); + self.http = try .init(allocator, io, &self.client, .{ .protocol = .http, .retry = retry }); + errdefer self.http.deinit(); + var path_buffer: [128]u8 = undefined; + const path = try self.server.path(&path_buffer); + self.file = try std.Io.Dir.openFile(.cwd(), self.http.io(), path, .{ .mode = .read_only }); + } + + fn deinit(self: *Fixture) void { + self.file.close(self.http.io()); + self.http.deinit(); + self.client.deinit(); + self.server.deinit(std.testing.io); + } +}; + +/// Reads `object` back through the fixture's file with `readers` concurrent +/// positional reads of `read_size` bytes, reader `i` at `i * read_size`, +/// all released at once. Fails on the first reader that failed or read +/// other bytes. +fn expectConcurrentReads(fixture: *Fixture, object: []const u8, comptime readers: usize, comptime read_size: usize) !void { + const Reader = struct { + fn run(file: std.Io.File, file_io: std.Io, output: []u8, offset: u64, result: *anyerror!usize, start: *std.Io.Event) void { + start.wait(file_io) catch |err| { + result.* = err; + return; + }; + result.* = file.readPositional(file_io, &.{output}, offset); + } + }; + const io = std.testing.io; + var outputs: [readers][read_size]u8 = undefined; + var results: [readers]anyerror!usize = undefined; + var released: std.Io.Event = .unset; + var group: std.Io.Group = .init; + defer group.cancel(io); + for (&outputs, &results, 0..) |*output, *result, index| { + try group.concurrent(io, Reader.run, .{ fixture.file, fixture.http.io(), output, index * read_size, result, &released }); + } + released.set(io); + try group.await(io); + + for (&outputs, results, 0..) |*output, result, index| { + try std.testing.expectEqual(read_size, try result); + try std.testing.expectEqualSlices(u8, object[index * read_size ..][0..read_size], output); + } +} + +test "generic HTTP keeps a large scattered positional read to one GET" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const offset = 13; + const request_size = 17 * 1024 * 1024 + 257; + + const object = try allocator.alloc(u8, offset + request_size); + defer allocator.free(object); + @memset(object, 0xa5); + const output = try allocator.alloc(u8, request_size); + defer allocator.free(output); + + var fixture: Fixture = undefined; + try fixture.init(object, .{}, .{}); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + // Fresh stats for the read: the open's HEAD is a request too. + stats.* = .{}; + const buffers = [_][]u8{ + output[0 .. 4 * 1024 * 1024], + output[4 * 1024 * 1024 .. 10 * 1024 * 1024], + output[10 * 1024 * 1024 ..], + }; + try std.testing.expectEqual( + request_size, + try fixture.file.readPositional(fixture.http.io(), &buffers, offset), + ); + try std.testing.expectEqualSlices(u8, object[offset..], output); + + try server.stop(io); + + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); +} + +test "generic HTTP follows a relative redirect chain at open and on every read" { + const io = std.testing.io; + + var object: [4096]u8 = undefined; + for (&object, 0..) |*byte, i| byte.* = @truncate(i); + var output: [1000]u8 = undefined; + + // `/object` redirects twice, and the second location resolves against + // the first hop's path. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .redirects = &.{ + .{ .path = "/object", .location = "/hop/next" }, + .{ .path = "/hop/next", .location = "../moved" }, + }, + .objects = &.{.{ .path = "/moved", .bytes = &object }}, + }, .{}); + defer fixture.deinit(); + const server = &fixture.server; + const http_io = fixture.http.io(); + + try std.testing.expectEqual(@as(u64, object.len), (try fixture.file.stat(http_io)).size); + const buffers = [_][]u8{&output}; + try std.testing.expectEqual(output.len, try fixture.file.readPositional(http_io, &buffers, 77)); + try std.testing.expectEqualSlices(u8, object[77..][0..output.len], &output); + + try server.stop(io); + // Only the object answers with bytes; the hops before it redirect. + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.get_requests.load(.acquire)); +} + +test "generic HTTP retries serially and reports aggregate retry counters" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const request_size = 2 * 1024 * 1024; + + const object = try allocator.alloc(u8, request_size); + defer allocator.free(object); + @memset(object, 0x3c); + const output = try allocator.alloc(u8, request_size); + defer allocator.free(output); + + var fixture: Fixture = undefined; + try fixture.init(object, .{ .fail_first_gets = 1 }, .{ + .max_retries = 1, + .initial_delay = .fromNanoseconds(0), + .max_delay = .fromNanoseconds(0), + }); + defer fixture.deinit(); + const server = &fixture.server; + const http_io = fixture.http.io(); + const stats = &fixture.http.governor.stats; + + stats.* = .{}; + const one_buffer = [_][]u8{output}; + try std.testing.expectEqual( + request_size, + try fixture.file.readPositional(http_io, &one_buffer, 0), + ); + try std.testing.expectEqual(@as(u64, 2), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); + + stats.* = .{}; + try std.testing.expectEqual( + request_size, + try fixture.file.readPositional(http_io, &one_buffer, 0), + ); + try std.testing.expectEqual(@as(u64, 1), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expectEqualSlices(u8, object, output); + + try server.stop(io); + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); +} + +test "generic HTTP physical concurrency does not exceed caller admission" { + const io = std.testing.io; + const admission = 3; + const read_size = 1024; + + var object: [admission * read_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index); + + var fixture: Fixture = undefined; + try fixture.init(&object, .{ .barrier_gets = admission }, .{}); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + stats.* = .{}; + try expectConcurrentReads(&fixture, &object, admission, read_size); + + try server.stop(io); + + const peak = server.peak_gets.load(.acquire); + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, admission), peak); + try std.testing.expect(peak <= admission); + try std.testing.expectEqual( + @as(usize, admission), + server.get_requests.load(.acquire), + ); + try std.testing.expectEqual(@as(u64, admission), stats.physical_requests.load(.acquire)); +} + +test "the shared range loop prepares the request once per attempt" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const request_size = 3 * 1024; + + var object: [request_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 7); + var output: [request_size]u8 = undefined; + + var server = try MockServer.init(io, &object, .{ .fail_first_gets = 2 }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + // An S3-style request: as a SigV4 signature must be, it is prepared + // again by the hook for every attempt. + const Signed = struct { + uri: std.Uri, + prepared: usize = 0, + + fn prepare(self: *@This(), _: vfs_request.Attempt, range: std.http.Header) anyerror!range_read.PreparedRequest { + try std.testing.expectEqualStrings("Range", range.name); + self.prepared += 1; + return .{ .uri = self.uri }; + } + }; + var url_buffer: [160]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + var signed: Signed = .{ .uri = try .parse(url) }; + const spec: range_read.RangeSpec = .{ + .request = .{ .target = url, .key = vfs_request.authorityOf(signed.uri) }, + }; + const buffers = [_][]u8{ output[0..1024], output[1024..] }; + + // No retry budget: one attempt, one hook call, the failure is counted. + var no_retry: vfs_request.Governor = .init(io, &client, "test", .throttle, instantRetries(0)); + try std.testing.expectError( + error.RetriesExhausted, + range_read.performRangeRead(&no_retry, spec, &signed, Signed.prepare, &buffers, 0, object.len), + ); + try std.testing.expectEqual(@as(usize, 1), signed.prepared); + try std.testing.expectEqual(@as(u64, 1), no_retry.stats.server_failures.load(.acquire)); + + signed.prepared = 0; + var one_retry: vfs_request.Governor = .init(io, &client, "test", .throttle, instantRetries(1)); + try std.testing.expectEqual( + object.len, + try range_read.performRangeRead(&one_retry, spec, &signed, Signed.prepare, &buffers, 0, object.len), + ); + try std.testing.expectEqual(@as(usize, 2), signed.prepared); + try std.testing.expectEqualSlices(u8, &object, &output); + + try server.stop(io); + + // One GET per hook call. + try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); + const stats = &one_retry.stats; + try std.testing.expectEqual(@as(u64, 2), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); +} + +test "one throttled GET holds every reader of the backend" { + const io = std.testing.io; + const readers = 8; + const read_size = 512; + + var object: [readers * read_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index); + + // Every reader's first GET is throttled, all eight answered together (the + // barrier), with a one second reset. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .barrier_gets = readers, + .throttle = .{ .first_gets = readers, .retry_after_s = 1 }, + }, fastRetry(50, 300_000)); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + try expectConcurrentReads(&fixture, &object, readers, read_size); + + try server.stop(io); + + // One hold covered every reader, the server-named second was honoured, + // and no reader burned a retry for it. + try std.testing.expectEqual(@as(u64, 1), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, readers), stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(usize, 2 * readers), server.get_requests.load(.acquire)); + const throttled_at = server.first_throttle_ns.load(.acquire); + const resumed_at = server.first_get_after_throttle_ns.load(.acquire); + try std.testing.expect(throttled_at != 0 and resumed_at != 0); + try std.testing.expect(resumed_at - throttled_at >= std.time.ns_per_s); +} + +test "a rate window is absorbed by the hold instead of the retry budget" { + const io = std.testing.io; + const readers = 12; + const read_size = 256; + + var object: [readers * read_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 3); + + // Four GETs per 100 ms; twelve readers need three windows at best. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .throttle = .{ .window = .{ .gets = 4, .per_ms = 100 } }, + }, fastRetry(50, 300_000)); + defer fixture.deinit(); + const stats = &fixture.http.governor.stats; + + try expectConcurrentReads(&fixture, &object, readers, read_size); + + try fixture.server.stop(io); + + try std.testing.expect(stats.holds.load(.acquire) > 0); + try std.testing.expect(stats.throttles.load(.acquire) > 0); + // A limited server never exhausts a reader's retries: the hold carries + // the whole episode. + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expect(stats.hold_wait_ns.load(.acquire) > 0); +} + +test "a throttled HEAD at open waits the hold out and succeeds" { + const io = std.testing.io; + + var object: [64]u8 = undefined; + @memset(&object, 0x11); + + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .throttle = .{ .first_heads = 1, .retry_after_s = 0 }, + }, fastRetry(20, 300_000)); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + try std.testing.expectEqual(@as(u64, object.len), try fixture.file.length(fixture.http.io())); + + try server.stop(io); + + // The open retried its HEAD after the hold, and the hold is the + // backend's, not the read path's. + try std.testing.expectEqual(@as(usize, 2), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); +} + +test "a 503 holds an object store and only retries a plain HTTP server" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const read_size = 512; + + var object: [read_size]u8 = undefined; + @memset(&object, 0x7e); + var output: [read_size]u8 = undefined; + + // Each backend's first two GETs are answered 503. + var server = try MockServer.init(io, &object, .{ + .throttle = .{ .first_gets = 2, .status = .service_unavailable }, + .objects = &.{ + .{ .path = "/bucket/object", .bytes = &object }, + .{ .path = "/bucket/object?alt=media", .bytes = &object }, + }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(5) }); + defer s3.deinit(); + var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint, .retry = instantRetries(5) }); + defer gcs.deinit(); + var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(5) }); + defer http.deinit(); + + var path_buffer: [128]u8 = undefined; + const backends = [_]struct { std.Io, *const vfs_request.Governor, []const u8, bool }{ + .{ s3.io(), &s3.governor, "bucket/object", true }, + .{ gcs.io(), &gcs.governor, "bucket/object", true }, + .{ http.io(), &http.governor, try server.path(&path_buffer), false }, + }; + for (backends) |backend| { + const backend_io, const governor, const path, const object_store = backend; + const file = try std.Io.Dir.openFile(.cwd(), backend_io, path, .{}); + defer file.close(backend_io); + server.get_requests.store(0, .release); + try std.testing.expectEqual(read_size, try file.readPositional(backend_io, &.{&output}, 0)); + + const stats = &governor.stats; + if (object_store) { + // A 503 is rate limiting. One reader, two throttles in a row: + // each arms its own hold once the previous one has passed, and + // neither is charged a retry. + try std.testing.expectEqual(@as(u64, 2), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + } else { + // A 503 is a server failure: no hold, one charged retry per + // attempt. + try std.testing.expectEqual(@as(u64, 0), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.retries.load(.acquire)); + } + } + + try server.stop(io); +} + +test "a server that never lets up fails the request with RateLimited" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const read_size = 256; + + var object: [read_size]u8 = undefined; + @memset(&object, 0x33); + var output: [read_size]u8 = undefined; + + var server = try MockServer.init(io, &object, .{ + .throttle = .{ .always = true, .retry_after_s = 0 }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + var url_buffer: [160]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + const uri: std.Uri = try .parse(url); + const buffers = [_][]u8{&output}; + + var governor: vfs_request.Governor = .init(io, &client, "test", .throttle, .{ + .max_retries = 5, + .initial_delay = .fromMilliseconds(20), + .max_delay = .fromMilliseconds(20), + .max_hold = .fromMilliseconds(50), + .throttle_budget = .fromMilliseconds(300), + }); + const started: std.Io.Timestamp = .now(io, .awake); + // The read never runs: the backend is rate limited past its budget. + // `fileReadPositional` maps this to `Unexpected` with its own log line. + try std.testing.expectError(error.RateLimited, range_read.performRangeRead(&governor, .{ + .request = .{ .target = url }, + }, &uri, range_read.static, &buffers, 0, object.len)); + const elapsed = started.durationTo(.now(io, .awake)).nanoseconds; + try std.testing.expect(elapsed >= 300 * std.time.ns_per_ms); + try std.testing.expect(elapsed < 5 * std.time.ns_per_s); + try std.testing.expect(governor.stats.holds.load(.acquire) > 1); + + try server.stop(io); +} + +test "a reader cancelled while the backend is held returns Canceled" { + const io = std.testing.io; + const read_size = 256; + + var object: [read_size]u8 = undefined; + @memset(&object, 0x21); + var output: [read_size]u8 = undefined; + + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .throttle = .{ .first_gets = 1, .retry_after_s = 30 }, + }, fastRetry(20, 300_000)); + defer fixture.deinit(); + + var failure: std.atomic.Value(u16) = .init(0); + var readers: std.Io.Group = .init; + try readers.concurrent(io, struct { + fn run(file_: std.Io.File, io_: std.Io, out: []u8, failure_: *std.atomic.Value(u16)) void { + const buffers = [_][]u8{out}; + _ = file_.readPositional(io_, &buffers, 0) catch |err| { + failure_.store(@intFromError(err), .release); + return; + }; + } + }.run, .{ fixture.file, fixture.http.io(), output[0..], &failure }); + + // Let the first GET be throttled and the reader settle into the hold. + try io.sleep(.fromMilliseconds(50), .awake); + readers.cancel(io); + try std.testing.expectEqual(@intFromError(error.Canceled), failure.load(.acquire)); + + try fixture.server.stop(io); +} + +test "generic HTTP cancels a GET that receives nothing and retries it as a timeout" { + const io = std.testing.io; + + var object: [4096]u8 = undefined; + for (&object, 0..) |*byte, i| byte.* = @truncate(i); + var output: [1000]u8 = undefined; + + // The first GET waits unanswered for the second: a peer that receives + // the request and sends nothing. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ .barrier_gets = 2 }, .{ + .initial_delay = .fromNanoseconds(0), + .max_delay = .fromNanoseconds(0), + .stall_timeout = .fromMilliseconds(100), + }); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + stats.* = .{}; + const buffers = [_][]u8{&output}; + try std.testing.expectEqual(output.len, try fixture.file.readPositional(fixture.http.io(), &buffers, 77)); + try std.testing.expectEqualSlices(u8, object[77..][0..output.len], &output); + try std.testing.expectEqual(@as(u64, 1), stats.timeouts.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); + + // The first GET answers late, to a connection the client closed: that + // write may fail. + server.stop(io) catch |err| if (err != error.WriteFailed) return err; + try std.testing.expectEqual(@as(usize, 2), server.get_requests.load(.acquire)); +} + +test "http, s3 and gs fail open, stat and access of an absent or denied object alike" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Case = struct { name: []const u8, status: std.http.Status, expected: anyerror }; + const cases = [_]Case{ + .{ .name = "missing", .status = .not_found, .expected = error.FileNotFound }, + .{ .name = "gone", .status = .gone, .expected = error.FileNotFound }, + .{ .name = "unauthorized", .status = .unauthorized, .expected = error.PermissionDenied }, + .{ .name = "forbidden", .status = .forbidden, .expected = error.PermissionDenied }, + }; + // The target each backend sends for an object: HTTP its path, S3 + // `//`, GCS the same with `?alt=media`. + const targets = [_]struct { []const u8, []const u8 }{ .{ "/", "" }, .{ "/bucket/", "" }, .{ "/bucket/", "?alt=media" } }; + const refusals = comptime refusals: { + var list: [cases.len * targets.len]MockServer.Refusal = undefined; + for (cases, 0..) |case, i| { + for (targets, 0..) |target, j| { + list[i * targets.len + j] = .{ .path = target[0] ++ case.name ++ target[1], .status = case.status }; + } + } + break :refusals list; + }; + + var object: [32]u8 = undefined; + @memset(&object, 0x6b); + var server = try MockServer.init(io, &object, .{ + .objects = &.{ + .{ .path = "/bucket/object", .bytes = &object }, + .{ .path = "/bucket/object?alt=media", .bytes = &object }, + }, + .refusals = &refusals, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http }); + defer http.deinit(); + var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1" }); + defer s3.deinit(); + var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint }); + defer gcs.deinit(); + + var authority_buffer: [64]u8 = undefined; + const backends = [_]struct { std.Io, []const u8 }{ + .{ http.io(), try std.fmt.bufPrint(&authority_buffer, "{s}/", .{endpoint["http://".len..]}) }, + .{ s3.io(), "bucket/" }, + .{ gcs.io(), "bucket/" }, + }; + for (backends) |backend| { + const backend_io, const prefix = backend; + var path_buffer: [128]u8 = undefined; + const present = try std.fmt.bufPrint(&path_buffer, "{s}object", .{prefix}); + try std.Io.Dir.access(.cwd(), backend_io, present, .{}); + try std.testing.expectEqual(@as(u64, object.len), (try std.Io.Dir.statFile(.cwd(), backend_io, present, .{})).size); + for (cases) |case| { + const path = try std.fmt.bufPrint(&path_buffer, "{s}{s}", .{ prefix, case.name }); + try std.testing.expectError(case.expected, std.Io.Dir.openFile(.cwd(), backend_io, path, .{})); + try std.testing.expectError(case.expected, std.Io.Dir.statFile(.cwd(), backend_io, path, .{})); + try std.testing.expectError(case.expected, std.Io.Dir.access(.cwd(), backend_io, path, .{})); + } + } + + try server.stop(io); +} + +test "http, s3 and gs fail a refused streaming read with InputOutput and seek back no further than 0" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + var object: [64]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 5 +% 1); + // Each backend's first GET is answered 500, and none has a retry left. + var server = try MockServer.init(io, &object, .{ + .fail_first_gets = 3, + .objects = &.{ + .{ .path = "/bucket/object", .bytes = &object }, + .{ .path = "/bucket/object?alt=media", .bytes = &object }, + }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(0) }); + defer http.deinit(); + var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(0) }); + defer s3.deinit(); + var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint, .retry = instantRetries(0) }); + defer gcs.deinit(); + + var path_buffer: [128]u8 = undefined; + const backend_ios = [_]std.Io{ http.io(), s3.io(), gcs.io() }; + const paths = [_][]const u8{ try server.path(&path_buffer), "bucket/object", "bucket/object" }; + var files: [backend_ios.len]std.Io.File = undefined; + var opened: usize = 0; + defer for (files[0..opened], backend_ios[0..opened]) |file, backend_io| file.close(backend_io); + for (&files, backend_ios, paths) |*file, backend_io, path| { + file.* = try std.Io.Dir.openFile(.cwd(), backend_io, path, .{}); + opened += 1; + } + + var head: [8]u8 = undefined; + const head_slices = [_][]u8{&head}; + for (files, backend_ios) |file, backend_io| { + try std.testing.expectError(error.InputOutput, file.readStreaming(backend_io, &head_slices)); + } + for (files, backend_ios) |file, backend_io| { + // Forward, then back by more than the position: at 0. + try backend_io.vtable.fileSeekBy(backend_io.userdata, file, 5); + try backend_io.vtable.fileSeekBy(backend_io.userdata, file, -9); + try std.testing.expectEqual(head.len, try file.readStreaming(backend_io, &head_slices)); + try std.testing.expectEqualSlices(u8, object[0..head.len], &head); + // `minInt(i64)`, whose negation overflows, stops at 0 too. + try backend_io.vtable.fileSeekBy(backend_io.userdata, file, std.math.minInt(i64)); + try std.testing.expectEqual(head.len, try file.readStreaming(backend_io, &head_slices)); + try std.testing.expectEqualSlices(u8, object[0..head.len], &head); + } + + try server.stop(io); + try std.testing.expectEqual(@as(usize, 9), server.get_requests.load(.acquire)); +} diff --git a/vfs/mock_server.zig b/vfs/mock_server.zig new file mode 100644 index 00000000..9206f65b --- /dev/null +++ b/vfs/mock_server.zig @@ -0,0 +1,326 @@ +//! A minimal HTTP origin for the VFS and loader tests: one object per path, +//! range reads, and injectable failures and rate limiting. Exported as +//! `VFS.MockServer` so tests above this layer can serve a fixture over a +//! real socket; nothing outside a test build references it. + +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const MockServer = struct { + /// How the server rate limits. A throttled answer carries the status + /// and, when the test names one, a `Retry-After`. + const Throttle = struct { + /// The first N GETs are throttled. + first_gets: usize = 0, + /// The first N HEADs are throttled. + first_heads: usize = 0, + /// Every GET is throttled, forever. + always: bool = false, + status: std.http.Status = .too_many_requests, + retry_after_s: ?u32 = null, + /// At most `gets` GETs per `per_ms` milliseconds; the rest are + /// throttled. + window: ?struct { gets: usize, per_ms: i64 } = null, + }; + + /// One extra path the server serves beside `/object`. + const Object = struct { path: []const u8, bytes: []const u8 }; + + /// A path answered with `status` and no object, on purpose. + pub const Refusal = struct { path: []const u8, status: std.http.Status }; + + /// A path answered with a 302 to `location`, for HEAD and GET alike. + pub const Redirect = struct { path: []const u8, location: []const u8 }; + + pub const Options = struct { + fail_first_gets: usize = 0, + barrier_gets: usize = 0, + throttle: Throttle = .{}, + objects: []const Object = &.{}, + /// Paths answered with a status of their own (404, 403, ...). + refusals: []const Refusal = &.{}, + redirects: []const Redirect = &.{}, + }; + + tcp: std.Io.net.Server, + /// The accept loop and its connections, from `start` to `stop`. + group: std.Io.Group = .init, + object: []const u8, + fail_first_gets: usize, + barrier_gets: usize, + throttle: Throttle, + objects: []const Object, + refusals: []const Refusal, + redirects: []const Redirect, + get_barrier: std.Io.Event = .unset, + head_requests: std.atomic.Value(usize) = .init(0), + get_requests: std.atomic.Value(usize) = .init(0), + active_gets: std.atomic.Value(usize) = .init(0), + peak_gets: std.atomic.Value(usize) = .init(0), + first_error: stdx.FirstError(anyerror) = .{}, + /// Awake nanoseconds of the first throttled answer and of the first + /// answer served after one; zero until they happen. + first_throttle_ns: std.atomic.Value(i64) = .init(0), + first_get_after_throttle_ns: std.atomic.Value(i64) = .init(0), + /// Guards the rate window. + window_mutex: std.Io.Mutex = .init, + window_started_ns: i64 = 0, + window_gets: usize = 0, + + pub fn init( + io: std.Io, + object: []const u8, + opts: Options, + ) !MockServer { + const address: std.Io.net.IpAddress = .{ .ip4 = .loopback(0) }; + return .{ + .tcp = try address.listen(io, .{ .reuse_address = true }), + .object = object, + .fail_first_gets = opts.fail_first_gets, + .barrier_gets = opts.barrier_gets, + .throttle = opts.throttle, + .objects = opts.objects, + .refusals = opts.refusals, + .redirects = opts.redirects, + }; + } + + /// Serves on a task of its own until `stop`. The server must not move + /// from here on. + pub fn start(self: *MockServer, io: std.Io) !void { + try self.group.concurrent(io, run, .{ self, io }); + } + + /// Stops serving, each connection canceled and joined, and returns the + /// first error the server met. + pub fn stop(self: *MockServer, io: std.Io) !void { + self.group.cancel(io); + if (self.first_error.get()) |err| return err; + } + + /// Stops serving if `stop` did not, and closes the listening socket. + pub fn deinit(self: *MockServer, io: std.Io) void { + self.group.cancel(io); + self.tcp.deinit(io); + } + + /// The path of `/object` for an `http` backend: `127.0.0.1:/object`. + pub fn path(self: *const MockServer, buffer: []u8) ![]const u8 { + return std.fmt.bufPrint(buffer, "127.0.0.1:{d}/object", .{self.port()}); + } + + fn bodyFor(self: *const MockServer, target: []const u8) ?[]const u8 { + if (std.mem.eql(u8, target, "/object")) return self.object; + for (self.objects) |object| { + if (std.mem.eql(u8, object.path, target)) return object.bytes; + } + return null; + } + + fn refusalFor(self: *const MockServer, target: []const u8) ?std.http.Status { + for (self.refusals) |refusal| { + if (std.mem.eql(u8, refusal.path, target)) return refusal.status; + } + return null; + } + + fn redirectFor(self: *const MockServer, target: []const u8) ?[]const u8 { + for (self.redirects) |redirect| { + if (std.mem.eql(u8, redirect.path, target)) return redirect.location; + } + return null; + } + + fn nowNs(io: std.Io) i64 { + return @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); + } + + /// Whether this GET is over the rate window. Counts only the requests + /// it lets through, so a throttled one does not consume the budget. + fn overWindow(self: *MockServer, io: std.Io) bool { + const window = self.throttle.window orelse return false; + self.window_mutex.lockUncancelable(io); + defer self.window_mutex.unlock(io); + const now = nowNs(io); + if (now - self.window_started_ns > window.per_ms * std.time.ns_per_ms) { + self.window_started_ns = now; + self.window_gets = 0; + } + if (self.window_gets >= window.gets) return true; + self.window_gets += 1; + return false; + } + + fn respondThrottled(self: *MockServer, io: std.Io, request: *std.http.Server.Request) !void { + _ = self.first_throttle_ns.cmpxchgStrong(0, nowNs(io), .release, .monotonic); + var retry_after_buffer: [16]u8 = undefined; + try request.respond("slow down", .{ + .status = self.throttle.status, + .keep_alive = false, + .extra_headers = if (self.throttle.retry_after_s) |seconds| &.{.{ + .name = "Retry-After", + .value = try std.fmt.bufPrint(&retry_after_buffer, "{d}", .{seconds}), + }} else &.{}, + }); + } + + pub fn port(self: *const MockServer) u16 { + return self.tcp.socket.address.getPort(); + } + + fn run(self: *MockServer, io: std.Io) std.Io.Cancelable!void { + var connections: std.Io.Group = .init; + defer connections.cancel(io); + + while (true) { + const stream = self.tcp.accept(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + _ = self.first_error.set(err); + return; + }, + }; + connections.concurrent(io, onConnection, .{ self, stream, io }) catch |err| { + stream.close(io); + _ = self.first_error.set(err); + return; + }; + } + } + + fn onConnection( + self: *MockServer, + stream: std.Io.net.Stream, + io: std.Io, + ) std.Io.Cancelable!void { + defer stream.close(io); + self.handleConnection(stream, io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => _ = self.first_error.set(err), + }; + } + + fn handleConnection(self: *MockServer, stream: std.Io.net.Stream, io: std.Io) !void { + var read_buffer: [8 * 1024]u8 = undefined; + var reader = stream.reader(io, &read_buffer); + var write_buffer: [8 * 1024]u8 = undefined; + var writer = stream.writer(io, &write_buffer); + var server: std.http.Server = .init(&reader.interface, &writer.interface); + var request = try server.receiveHead(); + + if (self.redirectFor(request.head.target)) |location| { + try request.respond("", .{ + .status = .found, + .keep_alive = false, + .extra_headers = &.{.{ .name = "Location", .value = location }}, + }); + return; + } + + const body = self.bodyFor(request.head.target) orelse { + const refusal = self.refusalFor(request.head.target); + try request.respond("refused", .{ + .status = refusal orelse .not_found, + .keep_alive = false, + }); + if (refusal != null) return; + return error.UnexpectedTarget; + }; + + switch (request.head.method) { + .HEAD => { + const head_ordinal = self.head_requests.fetchAdd(1, .acq_rel) + 1; + if (head_ordinal <= self.throttle.first_heads) { + try self.respondThrottled(io, &request); + return; + } + try request.respond(body, .{ .keep_alive = false }); + }, + .GET => { + const range = try parseRequestRange(&request, body.len); + const get_ordinal = self.get_requests.fetchAdd(1, .acq_rel) + 1; + const active = self.active_gets.fetchAdd(1, .acq_rel) + 1; + _ = self.peak_gets.fetchMax(active, .acq_rel); + defer _ = self.active_gets.fetchSub(1, .release); + + if (self.barrier_gets != 0 and get_ordinal <= self.barrier_gets) { + if (get_ordinal == self.barrier_gets) self.get_barrier.set(io); + try self.get_barrier.wait(io); + } + + if (get_ordinal <= self.fail_first_gets) { + try request.respond("retry", .{ + .status = .internal_server_error, + .keep_alive = false, + }); + return; + } + + if (self.throttle.always or get_ordinal <= self.throttle.first_gets or self.overWindow(io)) { + try self.respondThrottled(io, &request); + return; + } + if (self.first_throttle_ns.load(.acquire) != 0) { + _ = self.first_get_after_throttle_ns.cmpxchgStrong(0, nowNs(io), .release, .monotonic); + } + + var content_range_buffer: [96]u8 = undefined; + const content_range = try std.fmt.bufPrint( + &content_range_buffer, + "bytes {d}-{d}/{d}", + .{ range.start, range.end, body.len }, + ); + try request.respond(body[range.start .. range.end + 1], .{ + .status = .partial_content, + .keep_alive = false, + .extra_headers = &.{ + .{ .name = "Content-Range", .value = content_range }, + }, + }); + }, + else => { + try request.respond("method not allowed", .{ + .status = .method_not_allowed, + .keep_alive = false, + }); + return error.UnexpectedMethod; + }, + } + } + + const Range = struct { + start: usize, + end: usize, + }; + + fn parseRequestRange(request: *const std.http.Server.Request, object_len: usize) !Range { + var value: ?[]const u8 = null; + var headers = request.iterateHeaders(); + while (headers.next()) |header| { + if (!std.ascii.eqlIgnoreCase(header.name, "Range")) continue; + if (value != null) return error.DuplicateRange; + value = header.value; + } + + const range_value = value orelse return error.MissingRange; + const prefix = "bytes="; + if (range_value.len <= prefix.len or + !std.ascii.eqlIgnoreCase(range_value[0..prefix.len], prefix)) + { + return error.InvalidRange; + } + const dash = std.mem.indexOfScalar(u8, range_value[prefix.len..], '-') orelse + return error.InvalidRange; + const absolute_dash = prefix.len + dash; + if (absolute_dash == prefix.len or absolute_dash + 1 == range_value.len) + return error.InvalidRange; + + const result: Range = .{ + .start = try std.fmt.parseInt(usize, range_value[prefix.len..absolute_dash], 10), + .end = try std.fmt.parseInt(usize, range_value[absolute_dash + 1 ..], 10), + }; + if (result.end < result.start or result.end >= object_len) return error.InvalidRange; + return result; + } +}; diff --git a/vfs/parallel_read.zig b/vfs/parallel_read.zig deleted file mode 100644 index 70c9d89d..00000000 --- a/vfs/parallel_read.zig +++ /dev/null @@ -1,211 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -pub const InitOpts = struct { - chunk_size: usize, - num_workers: usize, - queue_capacity: usize, - max_retries: usize, - retry_initial_delay: std.Io.Duration, - retry_max_delay: std.Io.Duration, -}; - -pub const Status = union(enum) { - success, - retry_after: ?std.Io.Duration, - - pub fn retry() Status { - return .{ .retry_after = null }; - } -}; - -pub const BatchState = struct { - pending: std.atomic.Value(u32), - err: std.atomic.Value(u16) = .init(0), - - pub fn failed(batch: *BatchState) bool { - return batch.err.load(.acquire) != 0; - } - - pub fn fail(batch: *BatchState, err: anyerror) void { - _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); - } - - pub fn anyError(batch: *BatchState) ?anyerror { - const err = batch.err.load(.acquire); - return if (err == 0) null else @errorFromInt(err); - } - - pub fn completeOne(batch: *BatchState, io: std.Io) void { - const previous = batch.pending.fetchSub(1, .release); - std.debug.assert(previous > 0); - if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); - } - - pub fn waitUncancelable(batch: *BatchState, io: std.Io) void { - while (true) { - const pending = batch.pending.load(.acquire); - if (pending == 0) return; - io.futexWaitUncancelable(u32, &batch.pending.raw, pending); - } - } -}; - -pub fn Pool(comptime Job: type) type { - return struct { - const Self = @This(); - - group: std.Io.Group = .init, - job_queue: std.Io.Queue(Job), - queue_buf: []Job, - client: *std.http.Client, - prng: std.Random.DefaultPrng, - chunk_size: usize, - max_retries: usize, - retry_initial_delay: std.Io.Duration, - retry_max_delay: std.Io.Duration, - - pub fn init(self: *Self, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { - stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); - stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); - stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); - stdx.debug.assert(opts.retry_initial_delay.nanoseconds >= 0, "Pool retry initial delay must not be negative", .{}); - stdx.debug.assert(opts.retry_max_delay.nanoseconds >= 0, "Pool retry max delay must not be negative", .{}); - stdx.debug.assert(opts.retry_max_delay.nanoseconds >= opts.retry_initial_delay.nanoseconds, "Pool retry max delay must be at least the initial delay", .{}); - - const queue_buf = try allocator.alloc(Job, opts.queue_capacity); - errdefer allocator.free(queue_buf); - - self.* = .{ - .client = client, - .chunk_size = opts.chunk_size, - .max_retries = opts.max_retries, - .retry_initial_delay = opts.retry_initial_delay, - .retry_max_delay = opts.retry_max_delay, - .prng = .init(blk: { - var seed: u64 = undefined; - io.random(@ptrCast(&seed)); - break :blk seed; - }), - .job_queue = .init(queue_buf), - .queue_buf = queue_buf, - }; - errdefer self.group.cancel(io); - - for (0..opts.num_workers) |_| { - try self.group.concurrent(io, worker, .{ self, io }); - } - } - - pub fn deinit(self: *Self, allocator: std.mem.Allocator, io: std.Io) void { - self.job_queue.close(io); - self.group.cancel(io); - - allocator.free(self.queue_buf); - } - - fn worker(pool: *Self, io: std.Io) std.Io.Cancelable!void { - while (true) { - const job = pool.job_queue.getOne(io) catch break; - - var attempt: usize = 0; - while (true) { - if (job.batch.state.failed()) break; - const status = job.perform(pool.client) catch |err| { - job.batch.state.fail(err); - break; - }; - - switch (status) { - .success => break, - .retry_after => |duration| { - if (attempt >= pool.max_retries) { - job.batch.state.fail(error.RetriesExhausted); - break; - } - - io.sleep(duration orelse pool.backoffDuration(attempt), .awake) catch { - job.batch.state.fail(error.RetriesExhausted); - break; - }; - - attempt += 1; - }, - } - continue; - } - - job.batch.state.completeOne(io); - } - } - - fn backoffDuration(pool: *Self, attempt: usize) std.Io.Duration { - // FullJitter algorithm - const max_delay_ns: i96 = @min( - pool.retry_max_delay.toNanoseconds(), - pool.retry_initial_delay.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), - ); - const delay_ns = pool.prng.random().intRangeAtMost(i96, 0, max_delay_ns); - return .fromNanoseconds(delay_ns); - } - }; -} - -pub const ContentRange = struct { - start: u64, - end: u64, - total: u64, -}; - -pub fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; -} - -pub fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { - if (offset >= file_size) return 0; - - var requested: usize = 0; - for (data) |buf| requested += buf.len; - return @intCast(@min(file_size - offset, requested)); -} - -pub fn readChunk( - reader: *std.Io.Reader, - content_range: ?ContentRange, - file_offset: u64, - data: []const []u8, - chunk_offset: usize, - chunk_len: usize, -) !void { - if (content_range) |cr| { - if (cr.start > file_offset) return error.InvalidContentRange; - if (cr.start < file_offset) { - try reader.discardAll(file_offset - cr.start); - } - } - - var remaining = chunk_len; - var skip = chunk_offset; - - for (data) |buf| { - if (skip >= buf.len) { - skip -= buf.len; - continue; - } - - const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; - try reader.readSliceAll(destination); - remaining -= destination.len; - if (remaining == 0) break; - skip = 0; - } -} diff --git a/vfs/range_read.zig b/vfs/range_read.zig new file mode 100644 index 00000000..920720e6 --- /dev/null +++ b/vfs/range_read.zig @@ -0,0 +1,263 @@ +//! The Range specifics of a governed request: the `Range` header, the +//! `Content-Range` validation and the scatter into the caller's buffers. +//! Everything else (the hold, the retries, the classification) is in +//! `request.zig`. + +const std = @import("std"); + +const request = @import("request.zig"); + +const ContentRange = struct { + start: u64, + end: u64, + total: u64, +}; + +/// The backend's part of one GET: the URI, the authorization value and up to +/// `max_extra_headers` headers. The loop appends `Range`. Every slice must +/// stay valid until the attempt completes, so hooks write into storage that +/// outlives the call (the backend's per-read context). +pub const PreparedRequest = struct { + uri: std.Uri, + authorization: std.http.Client.Request.Headers.Value = .default, + extra_headers: []const std.http.Header = &.{}, +}; + +pub const max_extra_headers = 4; + +/// One range read: what identifies it, and whether it follows a redirect. +pub const RangeSpec = struct { + request: request.RequestSpec, + /// The object stores and a resolved download URL never redirect a + /// read; a plain HTTP server may. + redirects: request.Redirects = .forbid, +}; + +/// The `prepare` of a request that is the same on every attempt: a plain +/// GET of `uri`. +pub fn static(uri: *const std.Uri, _: request.Attempt, _: std.http.Header) anyerror!PreparedRequest { + return .{ .uri = uri.* }; +} + +/// One `GET` with `Range: bytes=offset-(offset+size-1)` per attempt through +/// the governed loop, where `size` is what `data` holds of the `file_size` +/// bytes past `offset`; nothing is sent when it is 0. `prepare` is called +/// once per attempt: S3's SigV4 signature covers the `Range` header and a +/// timestamp, a bearer token may have expired during a hold, and a signed +/// download URL may need re-resolving after a 403. +/// The caller holds its read slot through every retry, hold and sleep; +/// backend-local parallel readers would multiply the caller's width and +/// defeat its memory bounds. +pub fn performRangeRead( + governor: *request.Governor, + spec: RangeSpec, + context: anytype, + comptime prepare: fn (@TypeOf(context), request.Attempt, std.http.Header) anyerror!PreparedRequest, + data: []const []u8, + offset: u64, + file_size: u64, +) !usize { + const size = readSize(file_size, offset, data); + if (size == 0) return 0; + + const RangeRead = struct { + context: @TypeOf(context), + redirects: request.Redirects, + range: std.http.Header, + data: []const []u8, + offset: u64, + size: usize, + progress: *std.atomic.Value(u64) = undefined, + + fn attempt(self: *@This(), current: request.Attempt) anyerror!request.Outcome(void) { + self.progress = current.progress; + const prepared = try prepare(self.context, current, self.range); + std.debug.assert(prepared.extra_headers.len <= max_extra_headers); + var headers: [max_extra_headers + 1]std.http.Header = undefined; + @memcpy(headers[0..prepared.extra_headers.len], prepared.extra_headers); + headers[prepared.extra_headers.len] = self.range; + return request.exchange(void, current, prepared.uri, .{ + .authorization = prepared.authorization, + .extra_headers = headers[0 .. prepared.extra_headers.len + 1], + .redirects = self.redirects, + }, self, consume); + } + + fn consume(self: *@This(), res: *std.http.Client.Response) anyerror!void { + // The head bytes are released when the body reader is taken. + const content_range = contentRange(res.head); + return readResponse(res.reader(&.{}), res.head.status, content_range, self.offset, self.data, self.size, self.progress); + } + }; + + var range_buffer: [64]u8 = undefined; + var state: RangeRead = .{ + .context = context, + .redirects = spec.redirects, + .range = .{ + .name = "Range", + .value = std.fmt.bufPrint( + &range_buffer, + "bytes={d}-{d}", + .{ offset, offset + @as(u64, @intCast(size - 1)) }, + ) catch unreachable, + }, + .data = data, + .offset = offset, + .size = size, + }; + try request.perform(void, governor, spec.request, &state, RangeRead.attempt); + governor.stats.recordSuccess(size); + return size; +} + +fn contentRange(head: std.http.Client.Response.Head) ?ContentRange { + var it = head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) return parseContentRange(header.value); + } + return null; +} + +fn parseContentRange(value: []const u8) ?ContentRange { + const prefix = "bytes "; + if (value.len < prefix.len or !std.ascii.eqlIgnoreCase(value[0..prefix.len], prefix)) return null; + const range_and_total = value[prefix.len..]; + const dash = std.mem.indexOfScalar(u8, range_and_total, '-') orelse return null; + const slash = dash + 1 + (std.mem.indexOfScalar(u8, range_and_total[dash + 1 ..], '/') orelse return null); + if (dash == 0 or slash == dash + 1 or slash + 1 == range_and_total.len) return null; + if (std.mem.indexOfScalar(u8, range_and_total[slash + 1 ..], '/') != null) return null; + + const result: ContentRange = .{ + .start = std.fmt.parseInt(u64, range_and_total[0..dash], 10) catch return null, + .end = std.fmt.parseInt(u64, range_and_total[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, range_and_total[slash + 1 ..], 10) catch return null, + }; + if (result.end < result.start or result.end >= result.total) return null; + return result; +} + +fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { + if (offset >= file_size) return 0; + + var requested: usize = 0; + for (data) |buf| requested +|= buf.len; + return @intCast(@min(file_size - offset, requested)); +} + +/// Validates the response against the requested range, discards the prefix +/// of a `200` that ignored `Range`, then scatters `read_size` bytes, counting +/// what it receives in `progress`. +fn readResponse( + reader: *std.Io.Reader, + status: std.http.Status, + content_range: ?ContentRange, + offset: u64, + data: []const []u8, + read_size: usize, + progress: ?*std.atomic.Value(u64), +) !void { + std.debug.assert(read_size != 0); + + const response_start = switch (status) { + .partial_content => blk: { + const cr = content_range orelse return error.InvalidContentRange; + const requested_end = std.math.add(u64, offset, read_size - 1) catch return error.InvalidContentRange; + if (cr.start > offset or cr.end < requested_end) return error.InvalidContentRange; + break :blk cr.start; + }, + .ok => 0, + else => return error.UnexpectedStatus, + }; + + var skip = offset - response_start; + while (skip != 0) { + const len = @min(skip, progress_slice); + try reader.discardAll(len); + count(progress, len); + skip -= len; + } + try readScatter(reader, data, read_size, progress); +} + +/// Bytes read between two progress counts. +const progress_slice = 1 << 20; + +fn count(progress: ?*std.atomic.Value(u64), len: u64) void { + if (progress) |counter| _ = counter.fetchAdd(len, .monotonic); +} + +fn readScatter(reader: *std.Io.Reader, data: []const []u8, len: usize, progress: ?*std.atomic.Value(u64)) !void { + var remaining = len; + for (data) |buf| { + var destination = buf[0..@min(remaining, buf.len)]; + remaining -= destination.len; + while (destination.len != 0) { + const slice = destination[0..@min(destination.len, progress_slice)]; + try reader.readSliceAll(slice); + count(progress, slice.len); + destination = destination[slice.len..]; + } + } + if (remaining != 0) return error.UnexpectedEndOfOutput; +} + +test "Content-Range parsing is strict" { + try std.testing.expectEqual( + ContentRange{ .start = 2, .end = 9, .total = 10 }, + parseContentRange("bytes 2-9/10").?, + ); + try std.testing.expectEqual( + ContentRange{ .start = 2, .end = 9, .total = 10 }, + parseContentRange("ByTeS 2-9/10").?, + ); + try std.testing.expect(parseContentRange("items 2-9/10") == null); + try std.testing.expect(parseContentRange("bytes 2-9/*") == null); + try std.testing.expect(parseContentRange("bytes 9-2/10") == null); + try std.testing.expect(parseContentRange("bytes 2-10/10") == null); + try std.testing.expect(parseContentRange("bytes 2-9/10 trailing") == null); +} + +test "range responses fill scatter buffers" { + var reader: std.Io.Reader = .fixed("23456789"); + var first: [2]u8 = undefined; + var second: [3]u8 = undefined; + try readResponse( + &reader, + .partial_content, + .{ .start = 2, .end = 9, .total = 10 }, + 3, + &.{ &first, &second }, + 5, + null, + ); + try std.testing.expectEqualStrings("34", &first); + try std.testing.expectEqualStrings("567", &second); +} + +test "200 responses that ignore Range are positioned and scattered" { + var reader: std.Io.Reader = .fixed("0123456789"); + var first: [1]u8 = undefined; + var second: [4]u8 = undefined; + try readResponse(&reader, .ok, null, 3, &.{ &first, &second }, 5, null); + try std.testing.expectEqualStrings("3", &first); + try std.testing.expectEqualStrings("4567", &second); +} + +test "partial responses require a covering Content-Range" { + var reader: std.Io.Reader = .fixed("3456"); + var output: [4]u8 = undefined; + try std.testing.expectError(error.InvalidContentRange, readResponse(&reader, .partial_content, null, 3, &.{&output}, 4, null)); + + reader = .fixed("4567"); + try std.testing.expectError( + error.InvalidContentRange, + readResponse(&reader, .partial_content, .{ .start = 4, .end = 7, .total = 10 }, 3, &.{&output}, 4, null), + ); + + reader = .fixed("345"); + try std.testing.expectError( + error.InvalidContentRange, + readResponse(&reader, .partial_content, .{ .start = 3, .end = 5, .total = 10 }, 3, &.{&output}, 4, null), + ); +} diff --git a/vfs/request.zig b/vfs/request.zig new file mode 100644 index 00000000..9365d460 --- /dev/null +++ b/vfs/request.zig @@ -0,0 +1,1212 @@ +//! One governed HTTP request loop for every backend: wait out the backend's +//! hold, send, classify. A throttle (429 everywhere, 503 on the object +//! stores) arms a backend-wide hold instead of charging the request's retry +//! budget, so every reader of that backend backs off together and the traffic +//! actually falls; every other retryable failure keeps the per-request +//! backoff. The hold is a deadline, not a permit: nothing is narrowed +//! permanently, recovery is the deadline passing, and a reader's width stays +//! the ceiling it chose, because this loop runs inside its reads. + +const std = @import("std"); + +const log = @import("base.zig").scopedLog(.@"zml/vfs/request"); + +/// The retry knobs of one remote backend: its `Options.retry`. +pub const Retry = struct { + /// Retries per request for failures that are not rate limiting. + max_retries: usize = 5, + initial_delay: std.Io.Duration = .fromMilliseconds(500), + /// Longest backoff before a request's next attempt after a failure that + /// is not a throttle, a server-named delay included. A throttle waits + /// out the backend's hold instead (`max_hold`). + max_delay: std.Io.Duration = .fromSeconds(30), + /// Longest hold one throttle may arm over every request of the backend, + /// a server-named delay included. + max_hold: std.Io.Duration = .fromSeconds(120), + /// A throttling episode with no clean window longer than this fails the + /// backend's requests. + throttle_budget: std.Io.Duration = .fromSeconds(300), + /// An exchange that receives nothing for this long is canceled and + /// retried as a timeout: a peer that stops sending without closing the + /// connection would otherwise hold its reader, and the load waiting for + /// it, forever. Body readers count what they receive + /// (`Attempt.progress`); an exchange that counts nothing (a HEAD, a token + /// exchange) must finish within it. + stall_timeout: std.Io.Duration = .fromSeconds(60), +}; + +/// What a hold covers. One governor holds one scope, its backend instance; +/// the authority is carried so that a per-authority map is a local change in +/// `Governor.holdFor`. +pub const Key = []const u8; + +/// The rate-limit scope of a request: its URI authority. Only the host is +/// taken, since `Governor.holdFor` ignores the key; a per-authority scope +/// would want the port beside it. +pub fn authorityOf(uri: std.Uri) Key { + const host = uri.host orelse return ""; + return switch (host) { + .raw => |raw| raw, + .percent_encoded => |encoded| encoded, + }; +} + +/// One scope's rate-limit state, owned by the governor's mutex. +const Hold = struct { + /// When requests may go again; null when nothing is held. + until: ?std.Io.Timestamp = null, + /// The length of the current hold: a clean window that long ends the + /// episode. + length: std.Io.Duration = .fromNanoseconds(0), + /// Holds armed since the episode began; the backoff base. + consecutive: usize = 0, + episode_started: ?std.Io.Timestamp = null, + /// The episode outlived `throttle_budget`; requests fail until a clean + /// window clears it. + exhausted: bool = false, +}; + +/// One per backend instance: what each of its requests needs, its +/// rate-limit state and its read statistics. +pub const Governor = struct { + pub const Stats = struct { + physical_requests: std.atomic.Value(u64) = .init(0), + physical_bytes: std.atomic.Value(u64) = .init(0), + retries: std.atomic.Value(u64) = .init(0), + transient_retries: std.atomic.Value(u64) = .init(0), + timeouts: std.atomic.Value(u64) = .init(0), + server_failures: std.atomic.Value(u64) = .init(0), + throttles: std.atomic.Value(u64) = .init(0), + retry_delay_ns: std.atomic.Value(u64) = .init(0), + /// Backend-wide holds armed by rate limiting, and the time requests + /// spent waiting them out. + holds: std.atomic.Value(u64) = .init(0), + hold_wait_ns: std.atomic.Value(u64) = .init(0), + + fn recordAttempt(self: *Stats) void { + _ = self.physical_requests.fetchAdd(1, .monotonic); + } + + pub fn recordSuccess(self: *Stats, request_size: usize) void { + _ = self.physical_bytes.fetchAdd(@intCast(request_size), .monotonic); + } + + fn recordFailure(self: *Stats, failure: ReadFailure) void { + switch (failure) { + .transient => _ = self.transient_retries.fetchAdd(1, .monotonic), + .timeout => _ = self.timeouts.fetchAdd(1, .monotonic), + .server_failure => _ = self.server_failures.fetchAdd(1, .monotonic), + .throttle => _ = self.throttles.fetchAdd(1, .monotonic), + } + } + + fn recordRetry(self: *Stats) void { + _ = self.retries.fetchAdd(1, .monotonic); + } + + fn recordRetryDelay(self: *Stats, delay: std.Io.Duration) void { + const delay_ns: u64 = @intCast(@max(delay.nanoseconds, 0)); + _ = self.retry_delay_ns.fetchAdd(delay_ns, .monotonic); + } + + /// One rate-limit hold armed over every request of this backend. + fn recordHold(self: *Stats) void { + _ = self.holds.fetchAdd(1, .monotonic); + } + + /// Time one request spent waiting a hold out. + fn recordHoldWait(self: *Stats, waited: std.Io.Duration) void { + _ = self.hold_wait_ns.fetchAdd(@intCast(@max(waited.nanoseconds, 0)), .monotonic); + } + }; + + io: std.Io, + client: *std.http.Client, + /// The backend's name, for log lines. + name: []const u8, + /// What a 503 means for this backend: rate limiting on the object stores + /// (AWS `SlowDown`, GCS), a server failure elsewhere. + unavailable: ReadFailure, + retry: Retry, + stats: Stats = .{}, + mutex: std.Io.Mutex = .init, + hold: Hold = .{}, + + pub fn init(io: std.Io, client: *std.http.Client, name: []const u8, unavailable: ReadFailure, retry: Retry) Governor { + std.debug.assert(retry.initial_delay.nanoseconds >= 0); + std.debug.assert(retry.max_delay.nanoseconds >= retry.initial_delay.nanoseconds); + std.debug.assert(retry.max_hold.nanoseconds >= retry.initial_delay.nanoseconds); + return .{ .io = io, .client = client, .name = name, .unavailable = unavailable, .retry = retry }; + } + + /// Logs the read totals at debug, once, from the backend's `deinit`; + /// nothing when the backend sent no request. Holds, retries and failures + /// are logged at warn as they happen. + pub fn logTotals(self: *const Governor) void { + const stats = &self.stats; + const requests = stats.physical_requests.load(.acquire); + if (requests == 0) return; + // Under the VFS's scope, `zml/vfs`. + std.log.scoped(.@"zml/vfs").debug("{s} read totals: requests={d}, bytes={Bi:.2}, retries={d}, transient_retries={d}, timeouts={d}, server_failures={d}, throttles={d}, holds={d}, hold_wait_ms={d:.3}, retry_delay_ms={d:.3}", .{ + self.name, + requests, + stats.physical_bytes.load(.acquire), + stats.retries.load(.acquire), + stats.transient_retries.load(.acquire), + stats.timeouts.load(.acquire), + stats.server_failures.load(.acquire), + stats.throttles.load(.acquire), + stats.holds.load(.acquire), + @as(f64, @floatFromInt(stats.hold_wait_ns.load(.acquire))) / std.time.ns_per_ms, + @as(f64, @floatFromInt(stats.retry_delay_ns.load(.acquire))) / std.time.ns_per_ms, + }); + } + + /// A failed open, stat or access of `path` as `OpenError` reports it. + pub fn openFailure(self: *const Governor, path: []const u8, err: anyerror) OpenError { + return switch (err) { + error.FileNotFound => error.FileNotFound, + error.PermissionDenied => error.PermissionDenied, + error.Canceled => error.Canceled, + // Logged by `exchange`. + error.RequestFailed => error.Unexpected, + error.OutOfMemory => error.SystemResources, + else => { + log.err("{s}: looking up {s} failed: {t}", .{ self.name, path, err }); + return error.Unexpected; + }, + }; + } + + /// A failed read of `target` at `offset` as a backend's read reports it: + /// a canceled task stays `Canceled`, since every wait in the governed + /// loop is a cancellation point; any other failure is logged once at + /// err, by `exchange` or here, and is `Unexpected`. + pub fn readFailure(self: *const Governor, target: []const u8, offset: u64, err: anyerror) error{ Canceled, Unexpected } { + return switch (err) { + error.Canceled => error.Canceled, + // Logged by `exchange`. + error.RequestFailed => error.Unexpected, + else => { + log.err("{s}: reading {s} at {d} failed: {t}", .{ self.name, target, offset, err }); + return error.Unexpected; + }, + }; + } + + /// The scope `key` belongs to. One generic `HTTP` instance serves every + /// host of its scheme and one `S3` spans buckets, so per-instance + /// over-reaches there; a per-authority scope replaces this body with a + /// map without touching the loop. + fn holdFor(self: *Governor, key: Key) *Hold { + _ = key; + return &self.hold; + } +}; + +/// One attempt as a backend's request hook sees it. +pub const Attempt = struct { + /// The backend and the request as `perform` was given them, which + /// `exchange` sends and logs with. + governor: *Governor, + spec: RequestSpec, + /// The status the previous attempt returned, for a hook that reacts to + /// one (an expired signed URL answering 403 after a long hold). + previous_status: ?std.http.Status = null, + /// Bytes received so far, for a hook whose body is long: `exchange` + /// cancels an exchange this counter does not move for + /// `Retry.stall_timeout`. + progress: *std.atomic.Value(u64), +}; + +/// How often a watched exchange's progress is checked, per stall timeout. +const stall_checks = 12; + +/// Identity of one request, for the logs and for the classification. +pub const RequestSpec = struct { + /// Object identity for log lines. + target: []const u8, + /// The rate-limit scope: the request URI's authority. + key: Key = "", + /// Statuses this request retries although they are not retryable in + /// general, because its hook can do something about them: a signed + /// download URL that expired during a hold answers 401 or 403, and the + /// hook re-resolves it. Charged to the retry budget. + retry_once: []const std.http.Status = &.{}, +}; + +pub const ReadFailure = enum { + transient, + timeout, + server_failure, + throttle, +}; + +pub const Failure = struct { + failure: ReadFailure, + /// The delay the server named, if any. + delay: ?std.Io.Duration = null, + status: ?std.http.Status = null, +}; + +/// What one attempt produced: a value, or a reason to try again. +pub fn Outcome(comptime T: type) type { + return union(enum) { + done: T, + retry: Failure, + }; +} + +/// The one request loop. `attemptFn` performs a single HTTP exchange (see +/// `exchange`); this drives the hold, the classification and the two budgets +/// around it. A throttle re-runs the attempt without charging a retry, after +/// the hold it armed; any other retryable failure, a stall +/// (`Retry.stall_timeout`) included, sleeps the per-request backoff and charges +/// one of `max_retries`. A cancel of the caller propagates. +pub fn perform( + comptime T: type, + governor: *Governor, + spec: RequestSpec, + context: anytype, + comptime attemptFn: fn (@TypeOf(context), Attempt) anyerror!Outcome(T), +) anyerror!T { + var charged: usize = 0; + var previous_status: ?std.http.Status = null; + while (true) { + try admit(governor, spec); + governor.stats.recordAttempt(); + var progress: std.atomic.Value(u64) = .init(0); + switch (try attemptFn(context, .{ .governor = governor, .spec = spec, .previous_status = previous_status, .progress = &progress })) { + .done => |value| return value, + .retry => |failure| { + governor.stats.recordFailure(failure.failure); + previous_status = failure.status; + if (failure.failure == .throttle) { + // Not charged to the retry budget: the hold is what + // reduces the traffic, and the episode budget bounds it. + try reportThrottle(governor, spec, failure.delay); + continue; + } + if (charged >= governor.retry.max_retries) return error.RetriesExhausted; + charged += 1; + governor.stats.recordRetry(); + const retry = governor.retry; + const delay = if (failure.delay) |named| + clampDelay(named, .fromNanoseconds(0), retry.max_delay) + else + fullJitterDelay(governor.io, retry.initial_delay, retry.max_delay, charged); + governor.stats.recordRetryDelay(delay); + try governor.io.sleep(delay, .awake); + }, + } + } +} + +/// Runs one exchange on its own unit of concurrency while this task watches +/// the attempt's progress, and cancels it after `Retry.stall_timeout` without +/// progress. The exchange returns a canceled read as `Canceled`, unlogged, +/// so the outcome of a stalled exchange is decided here, as a timeout. +/// Without a unit of concurrency the exchange runs unwatched. +fn watched( + comptime T: type, + context: anytype, + comptime exchangeFn: fn (@TypeOf(context), Attempt) anyerror!Outcome(T), + attempt: Attempt, +) anyerror!Outcome(T) { + const Run = struct { + result: anyerror!Outcome(T) = undefined, + done: std.Io.Event = .unset, + + fn run(self: *@This(), io: std.Io, context_: @TypeOf(context), attempt_: Attempt) void { + self.result = exchangeFn(context_, attempt_); + self.done.set(io); + } + }; + const governor = attempt.governor; + var run: Run = .{}; + var future = governor.io.concurrent(Run.run, .{ &run, governor.io, context, attempt }) catch |err| switch (err) { + error.ConcurrencyUnavailable => return exchangeFn(context, attempt), + }; + const stall_timeout = governor.retry.stall_timeout; + const check: std.Io.Timeout = .{ .duration = .{ + .raw = .fromNanoseconds(@divTrunc(stall_timeout.nanoseconds, stall_checks)), + .clock = .awake, + } }; + var seen = attempt.progress.load(.monotonic); + var quiet: usize = 0; + while (quiet < stall_checks) { + run.done.waitTimeout(governor.io, check) catch |err| switch (err) { + error.Timeout => { + const now = attempt.progress.load(.monotonic); + quiet = if (now == seen) quiet + 1 else 0; + seen = now; + continue; + }, + error.Canceled => { + future.cancel(governor.io); + // An exchange already past its last cancelation point + // finished: its value is the caller's, and the cancel + // reaches the caller at its next cancelation point. + if (run.result) |outcome| switch (outcome) { + .done => { + governor.io.recancel(); + return outcome; + }, + .retry => {}, + } else |_| {} + return error.Canceled; + }, + }; + future.await(governor.io); + return run.result; + } + future.cancel(governor.io); + // It may have finished while being canceled. + if (run.result) |outcome| switch (outcome) { + .done => return outcome, + .retry => {}, + } else |_| {} + log.warn("{s}: {s} received nothing for {d:.1}s; retrying", .{ governor.name, attempt.spec.target, @as(f64, @floatFromInt(stall_timeout.nanoseconds)) / std.time.ns_per_s }); + return .{ .retry = .{ .failure = .timeout } }; +} + +/// Waits out the scope's hold. Returns at once when nothing is held; fails +/// with `error.RateLimited` when the episode outlived its budget. Waiters +/// wake at the deadline plus their own jitter, so the readers of one backend +/// do not all fire in the same instant, and re-check, since the hold may have +/// been extended while they slept. +fn admit(governor: *Governor, spec: RequestSpec) error{ Canceled, RateLimited }!void { + while (true) { + const deadline = deadline: { + governor.mutex.lockUncancelable(governor.io); + defer governor.mutex.unlock(governor.io); + const hold = governor.holdFor(spec.key); + const now: std.Io.Timestamp = .now(governor.io, .awake); + refresh(hold, now); + if (hold.exhausted) return error.RateLimited; + const until = hold.until orelse return; + if (until.nanoseconds <= now.nanoseconds) return; + break :deadline until; + }; + const jitter = fullJitterDelay(governor.io, governor.retry.initial_delay, governor.retry.initial_delay, 0); + const started: std.Io.Timestamp = .now(governor.io, .awake); + try std.Io.Clock.Timestamp.wait(deadline.addDuration(jitter).withClock(.awake), governor.io); + governor.stats.recordHoldWait(started.durationTo(.now(governor.io, .awake))); + } +} + +/// Arms or extends the scope's hold after a throttle. `named` is the delay +/// the server asked for, if any; it is honoured up to `max_hold` and never +/// below `initial_delay`, because `Retry-After: 0` and Hugging Face's +/// `RateLimit; t=0` parse to zero, and a throttle charges no retry: a zero +/// hold would be a hot loop. +fn reportThrottle(governor: *Governor, spec: RequestSpec, named: ?std.Io.Duration) error{RateLimited}!void { + const retry = governor.retry; + governor.mutex.lockUncancelable(governor.io); + defer governor.mutex.unlock(governor.io); + const hold = governor.holdFor(spec.key); + const now: std.Io.Timestamp = .now(governor.io, .awake); + refresh(hold, now); + + if (hold.episode_started) |started| { + if (started.durationTo(now).nanoseconds > retry.throttle_budget.nanoseconds) { + hold.exhausted = true; + // The caller reports the failure; this is the one line that + // says why its request never ran. + log.warn("{s}: rate limited for longer than the throttle budget while reading {s}; giving up", .{ governor.name, spec.target }); + return error.RateLimited; + } + } else { + hold.episode_started = now; + } + + const holding = if (hold.until) |until| now.nanoseconds < until.nanoseconds else false; + if (!holding) { + hold.consecutive += 1; + const backoff = fullJitterDelay(governor.io, retry.initial_delay, retry.max_delay, hold.consecutive - 1); + var delay = backoff; + if (named) |value| { + if (value.nanoseconds > delay.nanoseconds) delay = value; + } + hold.length = clampDelay(delay, retry.initial_delay, retry.max_hold); + hold.until = now.addDuration(hold.length); + log.warn("{s}: {s} is rate limiting; holding its requests for {d:.3}s", .{ + governor.name, + spec.target, + @as(f64, @floatFromInt(hold.length.toNanoseconds())) / std.time.ns_per_s, + }); + governor.stats.recordHold(); + return; + } + // A request already in flight when the hold armed got its own answer, + // with a later reset: extend, never shorten. + const asked = named orelse return; + const capped = clampDelay(asked, retry.initial_delay, retry.max_hold); + const extended = now.addDuration(capped); + if (extended.nanoseconds > hold.until.?.nanoseconds) { + hold.until = extended; + hold.length = capped; + } +} + +/// A clean window as long as the hold itself, with no new throttle, ends the +/// episode: nothing else needs counting, since a throttle inside the window +/// would have re-armed `until`. +fn refresh(hold: *Hold, now: std.Io.Timestamp) void { + const until = hold.until orelse return; + if (now.nanoseconds < until.addDuration(hold.length).nanoseconds) return; + hold.* = .{}; +} + +fn clampDelay(value: std.Io.Duration, low: std.Io.Duration, high: std.Io.Duration) std.Io.Duration { + return .fromNanoseconds(std.math.clamp(value.nanoseconds, low.nanoseconds, high.nanoseconds)); +} + +/// The hops `Redirects.follow` takes: std's default. +pub const max_redirects = 3; + +/// What happens to a redirect: the client chases it (`max_redirects` hops), +/// `consume` receives it because `HeadChain` walks the chain itself, or it +/// fails the request. +pub const Redirects = enum { + follow, + surface, + forbid, + + fn behavior(self: Redirects) std.http.Client.Request.RedirectBehavior { + return switch (self) { + .follow => .init(max_redirects), + .surface => .unhandled, + .forbid => .not_allowed, + }; + } +}; + +pub const ExchangeOptions = struct { + method: std.http.Method = .GET, + authorization: std.http.Client.Request.Headers.Value = .default, + content_type: std.http.Client.Request.Headers.Value = .default, + extra_headers: []const std.http.Header = &.{}, + /// Sent as the whole request body; the method must allow one. The + /// writer uses it as its own buffer, so it must be mutable. + payload: ?[]u8 = null, + /// An open, stat or access: an absent or denied object fails the + /// request with its `OpenError` before `consume`, unlogged. + open: bool = false, + redirects: Redirects = .forbid, + /// A refusal that fails the request is logged with up to 1 KiB of its + /// body, where an OAuth endpoint says why, instead of with its head. + log_refusal_body: bool = false, +}; + +/// One HTTP exchange of `attempt`'s request: send, receive the head, +/// classify it, and hand a 2xx (or a redirect `Redirects.surface` hands +/// back) to `consume` while the request is alive. Retryable connection, +/// send and receive failures and every classified status become `.retry`; +/// an unclassified status or any other connect or head failure fails with +/// `error.RequestFailed`, logged at err, except that an open request +/// (`ExchangeOptions.open`) fails an absent or denied object with +/// `FileNotFound` or `PermissionDenied` first, unlogged. The exchange alone +/// is watched (`Retry.stall_timeout`); a governed request its hook runs +/// first (a re-resolve, a token refresh) is watched by its own exchange. +pub fn exchange( + comptime T: type, + attempt: Attempt, + uri: std.Uri, + options: ExchangeOptions, + context: anytype, + comptime consume: fn (@TypeOf(context), *std.http.Client.Response) anyerror!T, +) anyerror!Outcome(T) { + const Exchange = struct { + uri: std.Uri, + options: ExchangeOptions, + context: @TypeOf(context), + + fn run(self: @This(), current: Attempt) anyerror!Outcome(T) { + return roundTrip(T, current, self.uri, self.options, self.context, consume); + } + }; + return watched(T, Exchange{ .uri = uri, .options = options, .context = context }, Exchange.run, attempt); +} + +fn roundTrip( + comptime T: type, + attempt: Attempt, + uri: std.Uri, + options: ExchangeOptions, + context: anytype, + comptime consume: fn (@TypeOf(context), *std.http.Client.Response) anyerror!T, +) anyerror!Outcome(T) { + const spec = attempt.spec; + // A followed redirect keeps each hop's Location and merged path here, + // 8 KiB of each, as much as a `HeadChain` URL; `req.uri` points into it. + var redirect_buffer: [2 * max_redirects * 8 * 1024]u8 = undefined; + var req = attempt.governor.client.request(options.method, uri, .{ + .redirect_behavior = options.redirects.behavior(), + .headers = .{ + // Range offsets and Content-Length count the stored bytes. + .accept_encoding = .{ .override = "identity" }, + .authorization = options.authorization, + .content_type = options.content_type, + }, + .extra_headers = options.extra_headers, + }) catch |err| switch (err) { + error.Timeout => return retryable(T, attempt, "connect", err, .timeout), + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.NameServerFailure, + => return retryable(T, attempt, "connect", err, .transient), + // A stalled exchange that `watched` canceled, or the caller's cancel. + error.Canceled => return err, + error.OutOfMemory => return err, + else => return fatal(attempt, "connect", err), + }; + defer req.deinit(); + + // A request that was not sent whole leaves its connection unusable; + // `deinit` would otherwise pool it, its reader still `ready`. + if (options.payload) |payload| { + req.sendBodyComplete(payload) catch |err| switch (err) { + error.WriteFailed => { + req.connection.?.closing = true; + if (canceled(&req)) return error.Canceled; + return retryable(T, attempt, "send body", err, .transient); + }, + }; + } else { + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + req.connection.?.closing = true; + if (canceled(&req)) return error.Canceled; + return retryable(T, attempt, "send headers", err, .transient); + }, + }; + } + + var res = req.receiveHead(if (options.redirects == .follow) &redirect_buffer else &.{}) catch |err| { + // A failed read already closed it; a redirect re-sends on a new + // connection that a failure leaves unusable too (or null). + if (req.connection) |connection| connection.closing = true; + if (canceled(&req)) return error.Canceled; + switch (err) { + error.Timeout => return retryable(T, attempt, "receive headers", err, .timeout), + error.HttpConnectionClosing, + error.HttpRequestTruncated, + error.ReadFailed, + error.WriteFailed, + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.NameServerFailure, + => return retryable(T, attempt, "receive headers", err, .transient), + // A stalled exchange that `watched` canceled, or the caller's cancel. + error.Canceled => return err, + error.OutOfMemory => return err, + else => return fatal(attempt, "receive headers", err), + } + }; + + if (options.open) { + if (openStatus(res.head.status)) |err| return err; + } + const class = res.head.status.class(); + if (class != .success and !(class == .redirect and options.redirects == .surface)) { + for (spec.retry_once) |status| { + if (status != res.head.status) continue; + log.warn("{s}: {s} answered {d}; retrying once with a fresh request", .{ + attempt.governor.name, + spec.target, + @backingInt(res.head.status), + }); + return .{ .retry = .{ .failure = .transient, .status = res.head.status } }; + } + const failure = classifyStatus(res.head.status, attempt.governor.unavailable) orelse { + if (!options.log_refusal_body) { + log.err("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); + return error.RequestFailed; + } + // Reading the body invalidates the head's strings. + const status = res.head.status; + var body_buffer: [1024]u8 = undefined; + var body: std.Io.Writer = .fixed(&body_buffer); + // A longer body is cut, one that fails is logged as it arrived. + _ = res.reader(&.{}).streamRemaining(&body) catch { + if (canceled(&req)) return error.Canceled; + }; + // One line: an OAuth endpoint spreads its JSON over several. + std.mem.replaceScalar(u8, body.buffered(), '\n', ' '); + log.err("{s}: {s} answered {d}: {s}", .{ attempt.governor.name, spec.target, @backingInt(status), body.buffered() }); + return error.RequestFailed; + }; + log.warn("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); + return .{ .retry = .{ + .failure = failure, + .delay = serverRetryDelay(res.head), + .status = res.head.status, + } }; + } + + return .{ .done = consume(context, &res) catch |err| switch (err) { + error.EndOfStream, error.ReadFailed => { + if (canceled(&req)) return error.Canceled; + return retryable(T, attempt, "read body", err, .transient); + }, + else => return err, + } }; +} + +/// Whether the exchange's read or write failed because it was canceled: +/// std's stream keeps the cancel and reports a failed read or write. +fn canceled(req: *const std.http.Client.Request) bool { + const connection = req.connection orelse return false; + if (connection.stream_reader.err) |err| return err == error.Canceled; + if (connection.stream_writer.err) |err| return err == error.Canceled; + return false; +} + +fn retryable( + comptime T: type, + attempt: Attempt, + stage: []const u8, + err: anyerror, + failure: ReadFailure, +) Outcome(T) { + log.warn("{s}: {s} for {s} failed: {}", .{ attempt.governor.name, stage, attempt.spec.target, err }); + return .{ .retry = .{ .failure = failure } }; +} + +fn fatal(attempt: Attempt, stage: []const u8, err: anyerror) error{RequestFailed} { + log.err("{s}: {s} for {s} failed: {}", .{ attempt.governor.name, stage, attempt.spec.target, err }); + return error.RequestFailed; +} + +/// The URLs of a HEAD redirect chain walked by hand: std hands a HEAD its +/// redirect unfollowed. Three buffers in rotation: the current URL, the +/// `Location` it answered (copied out of a head that dies with the +/// response) with the path merged against it, and the resolved next URL. +pub const HeadChain = struct { + urls: [3][8 * 1024]u8 = undefined, + + pub const End = struct { + /// `start_url` itself, or a URL in the chain's buffers. + url: []const u8, + /// Its `Content-Length`, when it sent one. + size: ?u64, + }; + + /// Follows `start_url` through at most `max_hops` redirects, absolute + /// or relative, to the URL that answers a 2xx. Each hop is one governed + /// HEAD, so a server that rate limits it holds the backend as a + /// throttled GET would. `authorize` names the credentials each URL is + /// sent. An absent or denied object fails the walk + /// (`ExchangeOptions.open`). + pub fn resolve( + self: *HeadChain, + governor: *Governor, + target: []const u8, + start_url: []const u8, + max_hops: usize, + context: anytype, + comptime authorize: fn (@TypeOf(context), url: []const u8) std.http.Client.Request.Headers.Value, + ) !End { + const Hop = struct { + context: @TypeOf(context), + url: []const u8, + uri: std.Uri, + /// Where a redirect's `Location` is copied. + location: []u8, + + const Result = union(enum) { end: ?u64, redirect: usize }; + + fn attempt(hop: *@This(), current: Attempt) anyerror!Outcome(Result) { + return exchange(Result, current, hop.uri, .{ + .method = .HEAD, + .authorization = authorize(hop.context, hop.url), + .open = true, + .redirects = .surface, + }, hop, consume); + } + + fn consume(hop: *@This(), res: *std.http.Client.Response) anyerror!Result { + if (res.head.status.class() == .success) return .{ .end = res.head.content_length }; + const location = res.head.location orelse return error.HttpRedirectLocationMissing; + if (location.len > hop.location.len) return error.HttpRedirectLocationOversize; + @memcpy(hop.location[0..location.len], location); + return .{ .redirect = location.len }; + } + }; + + var url = start_url; + // `start_url` is in none of the buffers. + var current: usize = self.urls.len - 1; + var hops: usize = 0; + while (true) { + const uri: std.Uri = try .parse(url); + const scratch = (current + 1) % self.urls.len; + var hop: Hop = .{ .context = context, .url = url, .uri = uri, .location = &self.urls[scratch] }; + const location_len = switch (try perform(Hop.Result, governor, .{ .target = target, .key = authorityOf(uri) }, &hop, Hop.attempt)) { + .end => |size| return .{ .url = url, .size = size }, + .redirect => |len| len, + }; + + if (hops == max_hops) return error.TooManyHttpRedirects; + hops += 1; + const output = (current + 2) % self.urls.len; + var aux: []u8 = &self.urls[scratch]; + const resolved = try uri.resolveInPlace(location_len, &aux); + url = try std.fmt.bufPrint(&self.urls[output], "{f}", .{resolved}); + current = output; + } + } +}; + +/// Retry classification of a non-2xx status; null when the status is not +/// retried. `unavailable` is what a 503 means for the backend: rate limiting +/// on the object stores (AWS `SlowDown`, GCS), a server failure elsewhere. +fn classifyStatus(status: std.http.Status, unavailable: ReadFailure) ?ReadFailure { + return switch (status) { + .request_timeout => .timeout, + .too_many_requests => .throttle, + .service_unavailable => unavailable, + else => if (status.class() == .server_error) .server_failure else null, + }; +} + +/// What an open, stat or access of a remote path fails with (vfs.zig's +/// header); `openStatus` maps the statuses, and anything else is logged +/// once, by `exchange` or `Governor.openFailure`. +pub const OpenError = error{ FileNotFound, PermissionDenied, Canceled, SystemResources, Unexpected }; + +/// The error a status means for the path an open request +/// (`ExchangeOptions.open`) asked about; null when the status is not one. +fn openStatus(status: std.http.Status) ?error{ FileNotFound, PermissionDenied } { + return switch (status) { + .not_found, .gone => error.FileNotFound, + .unauthorized, .forbidden => error.PermissionDenied, + else => null, + }; +} + +/// The retry delay the server names, if any: `Retry-After` delta-seconds +/// (the HTTP-date form is not parsed and falls back to the jittered delay) +/// or the `t=` reset of the `RateLimit` header Hugging Face sends. +fn serverRetryDelay(head: std.http.Client.Response.Head) ?std.Io.Duration { + var it = head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Retry-After")) { + return delaySeconds(header.value) orelse continue; + } + if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { + var parts = std.mem.splitScalar(u8, header.value, ';'); + while (parts.next()) |part| { + const trimmed = std.mem.trim(u8, part, " \t"); + if (std.mem.startsWith(u8, trimmed, "t=")) return delaySeconds(trimmed[2..]) orelse continue; + } + } + } + return null; +} + +fn delaySeconds(value: []const u8) ?std.Io.Duration { + const seconds = std.fmt.parseInt(u32, std.mem.trim(u8, value, " \t"), 10) catch return null; + return .fromSeconds(seconds); +} + +fn fullJitterDelay( + io: std.Io, + initial: std.Io.Duration, + maximum: std.Io.Duration, + attempt: usize, +) std.Io.Duration { + const max_delay_ns: i96 = @min( + maximum.toNanoseconds(), + initial.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), + ); + if (max_delay_ns <= 0) return .fromNanoseconds(0); + + var seed: u64 = undefined; + io.random(@ptrCast(&seed)); + var prng: std.Random.DefaultPrng = .init(seed); + return .fromNanoseconds(prng.random().intRangeAtMost(i96, 0, max_delay_ns)); +} + +const test_spec: RequestSpec = .{ .target = "object" }; + +fn testGovernor(initial_ms: i64, max_delay_ms: i64, max_hold_ms: i64, budget_ms: i64) Governor { + // No attempt in these tests reaches the network: no client. + return .init(std.testing.io, undefined, "test", .throttle, .{ + .max_retries = 3, + .initial_delay = .fromMilliseconds(initial_ms), + .max_delay = .fromMilliseconds(max_delay_ms), + .max_hold = .fromMilliseconds(max_hold_ms), + .throttle_budget = .fromMilliseconds(budget_ms), + }); +} + +test "a throttle arms a hold of at least the initial delay, even when the server names zero" { + var governor = testGovernor(20, 20, 1000, 300_000); + + // `Retry-After: 0` and Hugging Face's `RateLimit; t=0` parse to zero. + try reportThrottle(&governor, test_spec, .fromSeconds(0)); + const hold = governor.hold; + try std.testing.expect(hold.until != null); + try std.testing.expectEqual(@as(i96, 20 * std.time.ns_per_ms), hold.length.nanoseconds); + try std.testing.expectEqual(@as(usize, 1), hold.consecutive); + try std.testing.expect(hold.episode_started != null); + try std.testing.expectEqual(@as(u64, 1), governor.stats.holds.load(.acquire)); +} + +test "a longer named delay extends a running hold and a shorter one does not" { + var governor = testGovernor(10, 10, 1000, 300_000); + + try reportThrottle(&governor, test_spec, null); + const armed = governor.hold.until.?; + + // A request that was already in flight got its own answer, with a later + // reset: the hold extends. + try reportThrottle(&governor, test_spec, .fromMilliseconds(500)); + const extended = governor.hold.until.?; + try std.testing.expect(extended.nanoseconds > armed.nanoseconds); + + // A shorter one never shortens it, and neither arms a second hold. + try reportThrottle(&governor, test_spec, .fromMilliseconds(1)); + try std.testing.expectEqual(extended.nanoseconds, governor.hold.until.?.nanoseconds); + try std.testing.expectEqual(@as(usize, 1), governor.hold.consecutive); + try std.testing.expectEqual(@as(u64, 1), governor.stats.holds.load(.acquire)); +} + +test "a named delay is capped by max_hold" { + var governor = testGovernor(10, 10, 50, 300_000); + + try reportThrottle(&governor, test_spec, .fromSeconds(3600)); + try std.testing.expectEqual(@as(i96, 50 * std.time.ns_per_ms), governor.hold.length.nanoseconds); +} + +test "a second throttle inside the clean window doubles the backoff base" { + var governor = testGovernor(10, 1000, 10_000, 300_000); + + try reportThrottle(&governor, test_spec, null); + try std.testing.expectEqual(@as(i96, 10 * std.time.ns_per_ms), governor.hold.length.nanoseconds); + + // The hold has passed but the clean window has not: rewind it by hand + // rather than sleeping through a race. + const now: std.Io.Timestamp = .now(std.testing.io, .awake); + governor.hold.until = now.subDuration(.fromMilliseconds(1)); + try reportThrottle(&governor, test_spec, null); + try std.testing.expectEqual(@as(usize, 2), governor.hold.consecutive); + // Full jitter over twice the base, floored at the initial delay. + try std.testing.expect(governor.hold.length.nanoseconds >= 10 * std.time.ns_per_ms); + try std.testing.expect(governor.hold.length.nanoseconds <= 20 * std.time.ns_per_ms); + try std.testing.expectEqual(@as(u64, 2), governor.stats.holds.load(.acquire)); +} + +test "a clean window as long as the hold ends the episode" { + var governor = testGovernor(10, 1000, 10_000, 300_000); + + try reportThrottle(&governor, test_spec, null); + const length = governor.hold.length; + const now: std.Io.Timestamp = .now(std.testing.io, .awake); + governor.hold.until = now.subDuration(length).subDuration(.fromMilliseconds(1)); + + try reportThrottle(&governor, test_spec, null); + try std.testing.expectEqual(@as(usize, 1), governor.hold.consecutive); + try std.testing.expectEqual(@as(i96, 10 * std.time.ns_per_ms), governor.hold.length.nanoseconds); +} + +test "an episode longer than the throttle budget fails requests until a clean window" { + var governor = testGovernor(10, 10, 10_000, 50); + + try reportThrottle(&governor, test_spec, .fromSeconds(30)); + const now: std.Io.Timestamp = .now(std.testing.io, .awake); + governor.hold.episode_started = now.subDuration(.fromMilliseconds(51)); + + try std.testing.expectError(error.RateLimited, reportThrottle(&governor, test_spec, null)); + try std.testing.expect(governor.hold.exhausted); + // Every other request of this backend fails at once, without waiting the + // hold out. + try std.testing.expectError(error.RateLimited, admit(&governor, test_spec)); + + // The next clean window clears the episode. + governor.hold.until = now.subDuration(governor.hold.length).subDuration(.fromMilliseconds(1)); + try admit(&governor, test_spec); + try std.testing.expect(governor.hold.until == null); + try std.testing.expect(!governor.hold.exhausted); +} + +test "admit returns at once when nothing is held" { + var governor = testGovernor(10, 10, 1000, 300_000); + + const started: std.Io.Timestamp = .now(std.testing.io, .awake); + try admit(&governor, test_spec); + try std.testing.expect(started.durationTo(.now(std.testing.io, .awake)).nanoseconds < 5 * std.time.ns_per_ms); + try std.testing.expectEqual(@as(u64, 0), governor.stats.hold_wait_ns.load(.acquire)); +} + +test "admit waits a hold out and counts the wait" { + var governor = testGovernor(1, 1, 1000, 300_000); + + try reportThrottle(&governor, test_spec, .fromMilliseconds(15)); + const started: std.Io.Timestamp = .now(std.testing.io, .awake); + try admit(&governor, test_spec); + try std.testing.expect(started.durationTo(.now(std.testing.io, .awake)).nanoseconds >= 10 * std.time.ns_per_ms); + try std.testing.expect(governor.stats.hold_wait_ns.load(.acquire) > 0); +} + +test "a waiter blocked on a hold is cancelable" { + const io = std.testing.io; + var governor = testGovernor(1, 1, 100_000, 300_000); + try reportThrottle(&governor, test_spec, .fromSeconds(60)); + + var result: std.atomic.Value(u16) = .init(0); + var started: std.Io.Event = .unset; + var group: std.Io.Group = .init; + try group.concurrent(io, struct { + fn run(governor_: *Governor, started_: *std.Io.Event, result_: *std.atomic.Value(u16)) void { + started_.set(governor_.io); + admit(governor_, test_spec) catch |err| { + result_.store(@intFromError(err), .release); + return; + }; + } + }.run, .{ &governor, &started, &result }); + try started.wait(io); + try io.sleep(.fromMilliseconds(5), .awake); + group.cancel(io); + try std.testing.expectEqual(@intFromError(error.Canceled), result.load(.acquire)); +} + +test "only rate limiting holds: a timeout and a server failure keep the per-request budget" { + var governor = testGovernor(0, 0, 1000, 300_000); + + const Attempts = struct { + seen: usize = 0, + fn run(self: *@This(), _: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + return switch (self.seen) { + 1 => .{ .retry = .{ .failure = .timeout } }, + 2 => .{ .retry = .{ .failure = .server_failure } }, + else => .{ .done = self.seen }, + }; + } + }; + var attempts: Attempts = .{}; + try std.testing.expectEqual(@as(usize, 3), try perform(usize, &governor, test_spec, &attempts, Attempts.run)); + try std.testing.expect(governor.hold.until == null); + try std.testing.expectEqual(@as(u64, 0), governor.stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), governor.stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 3), governor.stats.physical_requests.load(.acquire)); +} + +test "a throttle retries without charging the retry budget" { + var governor = testGovernor(0, 0, 1000, 300_000); + + const Attempts = struct { + seen: usize = 0, + fn run(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + // The retry budget is three; ten throttles do not exhaust it. + if (self.seen <= 10) return .{ .retry = .{ .failure = .throttle, .status = .too_many_requests } }; + try std.testing.expectEqual(@as(?std.http.Status, .too_many_requests), current.previous_status); + return .{ .done = self.seen }; + } + }; + var attempts: Attempts = .{}; + try std.testing.expectEqual(@as(usize, 11), try perform(usize, &governor, test_spec, &attempts, Attempts.run)); + try std.testing.expectEqual(@as(u64, 0), governor.stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 10), governor.stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 11), governor.stats.physical_requests.load(.acquire)); + try std.testing.expect(governor.stats.holds.load(.acquire) >= 1); +} + +test "the retry budget still bounds failures that are not rate limiting" { + var governor = testGovernor(0, 0, 1000, 300_000); + + const Attempts = struct { + seen: usize = 0, + fn run(self: *@This(), _: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + return .{ .retry = .{ .failure = .transient } }; + } + }; + var attempts: Attempts = .{}; + try std.testing.expectError(error.RetriesExhausted, perform(usize, &governor, test_spec, &attempts, Attempts.run)); + // The first attempt plus `max_retries` of them. + try std.testing.expectEqual(@as(usize, 4), attempts.seen); +} + +test "retry status classification is typed and 503 depends on the backend" { + try std.testing.expectEqual(ReadFailure.timeout, classifyStatus(.request_timeout, .server_failure).?); + try std.testing.expectEqual(ReadFailure.throttle, classifyStatus(.too_many_requests, .server_failure).?); + try std.testing.expectEqual(ReadFailure.server_failure, classifyStatus(.bad_gateway, .throttle).?); + try std.testing.expectEqual(ReadFailure.server_failure, classifyStatus(.service_unavailable, .server_failure).?); + try std.testing.expectEqual(ReadFailure.throttle, classifyStatus(.service_unavailable, .throttle).?); + try std.testing.expect(classifyStatus(.not_found, .throttle) == null); +} + +test "absent and denied statuses map to one error each" { + try std.testing.expect(openStatus(.not_found).? == error.FileNotFound); + try std.testing.expect(openStatus(.gone).? == error.FileNotFound); + try std.testing.expect(openStatus(.unauthorized).? == error.PermissionDenied); + try std.testing.expect(openStatus(.forbidden).? == error.PermissionDenied); + for ([_]std.http.Status{ .ok, .partial_content, .bad_request, .too_many_requests, .internal_server_error, .service_unavailable }) |status| { + try std.testing.expect(openStatus(status) == null); + } + + const governor = testGovernor(10, 10, 1000, 300_000); + try std.testing.expect(governor.openFailure("object", error.FileNotFound) == error.FileNotFound); + try std.testing.expect(governor.openFailure("object", error.PermissionDenied) == error.PermissionDenied); + try std.testing.expect(governor.openFailure("object", error.Canceled) == error.Canceled); + try std.testing.expect(governor.openFailure("object", error.OutOfMemory) == error.SystemResources); + try std.testing.expect(governor.openFailure("object", error.RequestFailed) == error.Unexpected); + try std.testing.expect(governor.openFailure("object", error.InvalidContentRange) == error.Unexpected); +} + +test "a failed read is Canceled or Unexpected" { + const governor = testGovernor(10, 10, 1000, 300_000); + try std.testing.expect(governor.readFailure("object", 0, error.Canceled) == error.Canceled); + try std.testing.expect(governor.readFailure("object", 0, error.RequestFailed) == error.Unexpected); + try std.testing.expect(governor.readFailure("object", 0, error.InvalidContentRange) == error.Unexpected); +} + +test "server retry delay comes from Retry-After seconds or a RateLimit reset" { + const Head = std.http.Client.Response.Head; + const retry_after = try Head.parse("HTTP/1.1 503 Service Unavailable\r\nretry-after: 3\r\n\r\n"); + try std.testing.expectEqual(std.Io.Duration.fromSeconds(3), serverRetryDelay(retry_after).?); + + const rate_limit = try Head.parse("HTTP/1.1 429 Too Many Requests\r\nRateLimit: \"default\"; r=0; t=7\r\n\r\n"); + try std.testing.expectEqual(std.Io.Duration.fromSeconds(7), serverRetryDelay(rate_limit).?); + + const http_date = try Head.parse("HTTP/1.1 503 Service Unavailable\r\nRetry-After: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"); + try std.testing.expect(serverRetryDelay(http_date) == null); + const none = try Head.parse("HTTP/1.1 500 Internal Server Error\r\n\r\n"); + try std.testing.expect(serverRetryDelay(none) == null); +} + +test "a server-named retry delay sleeps at most max_delay" { + var governor = testGovernor(1, 20, 20, 1000); + const Failing = struct { + seen: usize = 0, + fn attempt(self: *@This(), _: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + // A failing server that asks for an hour. + if (self.seen == 1) return .{ .retry = .{ .failure = .server_failure, .delay = .fromSeconds(3600) } }; + return .{ .done = self.seen }; + } + }; + var failing: Failing = .{}; + try std.testing.expectEqual(@as(usize, 2), try perform(usize, &governor, test_spec, &failing, Failing.attempt)); + try std.testing.expectEqual(@as(u64, 20 * std.time.ns_per_ms), governor.stats.retry_delay_ns.load(.acquire)); +} + +test "an attempt that receives nothing is canceled and retried as a timeout" { + var governor: Governor = .init(std.testing.io, undefined, "test", .throttle, .{ + .max_retries = 3, + .initial_delay = .fromMilliseconds(1), + .max_delay = .fromMilliseconds(1), + .max_hold = .fromMilliseconds(1), + .throttle_budget = .fromSeconds(1), + .stall_timeout = .fromMilliseconds(120), + }); + const Hang = struct { + seen: usize = 0, + progressed: u64 = 0, + // Watched as `exchange` watches its round trip. + fn attempt(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + return watched(usize, self, receive, current); + } + + fn receive(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + // The first exchange hangs, the second one keeps receiving and + // takes longer than the stall timeout overall. + if (self.seen == 1) { + try std.testing.io.sleep(.fromSeconds(60), .awake); + return error.TestUnexpectedResult; + } + for (0..6) |_| { + try std.testing.io.sleep(.fromMilliseconds(40), .awake); + _ = current.progress.fetchAdd(1, .monotonic); + self.progressed += 1; + } + return .{ .done = self.seen }; + } + }; + var hang: Hang = .{}; + try std.testing.expectEqual(@as(usize, 2), try perform(usize, &governor, test_spec, &hang, Hang.attempt)); + try std.testing.expectEqual(@as(u64, 6), hang.progressed); + try std.testing.expectEqual(@as(u64, 1), governor.stats.timeouts.load(.acquire)); +} + +test "an attempt waiting on a request of its own is not a stall" { + var governor: Governor = .init(std.testing.io, undefined, "test", .throttle, .{ + .max_retries = 3, + .initial_delay = .fromMilliseconds(1), + .max_delay = .fromMilliseconds(1), + .max_hold = .fromMilliseconds(1), + .throttle_budget = .fromSeconds(1), + .stall_timeout = .fromMilliseconds(120), + }); + const Nested = struct { + fn attempt(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + _ = try perform(usize, current.governor, test_spec, self, wait); + return .{ .done = 1 }; + } + + // A re-resolve or token refresh waiting out a hold, outside any + // exchange, for longer than the stall timeout. + fn wait(_: *@This(), _: Attempt) anyerror!Outcome(usize) { + try std.testing.io.sleep(.fromMilliseconds(300), .awake); + return .{ .done = 0 }; + } + }; + var nested: Nested = .{}; + try std.testing.expectEqual(@as(usize, 1), try perform(usize, &governor, test_spec, &nested, Nested.attempt)); + try std.testing.expectEqual(@as(u64, 0), governor.stats.timeouts.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), governor.stats.physical_requests.load(.acquire)); +} + +test "an exchange canceled while it waits for the head returns Canceled, not a retry" { + const MockServer = @import("mock_server.zig").MockServer; + const io = std.testing.io; + + // The GET waits unanswered for a second one that never comes. + var server = try MockServer.init(io, "object", .{ .barrier_gets = 2 }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = std.testing.allocator, .io = io }; + defer client.deinit(); + var governor: Governor = .init(io, &client, "test", .throttle, .{}); + var url_buffer: [64]u8 = undefined; + const uri: std.Uri = try .parse(try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()})); + + const Exchange = struct { + // Unwatched, so that the cancel lands in the read rather than in `watched`. + fn run(governor_: *Governor, uri_: std.Uri, result: *anyerror!Outcome(void)) void { + var progress: std.atomic.Value(u64) = .init(0); + result.* = roundTrip(void, .{ .governor = governor_, .spec = test_spec, .progress = &progress }, uri_, .{ + .extra_headers = &.{.{ .name = "Range", .value = "bytes=0-5" }}, + }, {}, consume); + } + + fn consume(_: void, _: *std.http.Client.Response) anyerror!void {} + }; + var result: anyerror!Outcome(void) = .{ .done = {} }; + var group: std.Io.Group = .init; + try group.concurrent(io, Exchange.run, .{ &governor, uri, &result }); + try io.sleep(.fromMilliseconds(50), .awake); + group.cancel(io); + try std.testing.expectError(error.Canceled, result); + + try server.stop(io); +} + +test "the request authority is the rate-limit scope" { + const uri: std.Uri = try .parse("https://huggingface.co/api/models/x"); + try std.testing.expectEqualStrings("huggingface.co", authorityOf(uri)); +} diff --git a/vfs/s3.zig b/vfs/s3.zig index eb849e21..65132a64 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -2,8 +2,10 @@ const std = @import("std"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/s3"); @@ -141,185 +143,26 @@ pub const AwsSigV4 = struct { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const S3 = struct { - const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - backend: *S3, - uri: std.Uri, - url: []const u8, - }; - - const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var timestamp_buf: [16]u8 = undefined; - const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch unreachable; - - const signer: AwsSigV4 = .{ - .access_key = job.batch.backend.config.access_key, - .secret_key = job.batch.backend.config.secret_key, - .region = job.batch.backend.config.region, - .service = job.batch.backend.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = signer.generateAuthHeader( - &authorization_buffer, - .GET, - job.batch.uri, - timestamp, - &.{.{ .name = "Range", .value = range_header }}, - ) catch |err| { - log.err("Failed to generate auth header: {}", .{err}); - return err; - }; - - var req = client.request(.GET, job.batch.uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - .{ .name = "Range", .value = range_header }, - }, - }) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout, .too_many_requests => .retry(), - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; - }; - - pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{ - .chunk_size = 16 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, - }; - - pub const Config = struct { + /// Without both keys, requests go unsigned. + pub const Options = struct { access_key: ?[]const u8 = null, secret_key: ?[]const u8 = null, endpoint_url: []const u8, region: []const u8, - auth_service: []const u8 = "s3", + retry: request.Retry = .{}, }; + /// The strings of the `Options` given to `init`, owned. + const Config = struct { + access_key: ?[]const u8, + secret_key: ?[]const u8, + endpoint_url: []const u8, + region: []const u8, + }; + + /// The SigV4 service every request is signed for. + const auth_service = "s3"; + const Handle = struct { pub const Type = enum { file, directory }; @@ -344,38 +187,24 @@ pub const S3 = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, - client: *std.http.Client, config: Config, - read_pool: *ParallelRead.Pool, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, - pub fn init( - allocator: std.mem.Allocator, - inner: std.Io, - http_client: *std.http.Client, - config: Config, - opts: InitOpts, - ) !S3 { - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !S3 { return .{ .allocator = allocator, .base = .init(inner), - .client = http_client, - .read_pool = read_pool, + // S3 rate limiting is `503 SlowDown` as well as `429`. + .governor = .init(inner, http_client, "s3", .throttle, options.retry), .config = .{ - .access_key = if (config.access_key) |k| try allocator.dupe(u8, k) else null, - .secret_key = if (config.secret_key) |k| try allocator.dupe(u8, k) else null, - .endpoint_url = try allocator.dupe(u8, config.endpoint_url), - .region = try allocator.dupe(u8, config.region), - .auth_service = try allocator.dupe(u8, config.auth_service), + .access_key = if (options.access_key) |k| try allocator.dupe(u8, k) else null, + .secret_key = if (options.secret_key) |k| try allocator.dupe(u8, k) else null, + .endpoint_url = try allocator.dupe(u8, options.endpoint_url), + .region = try allocator.dupe(u8, options.region), }, }; } @@ -422,13 +251,11 @@ pub const S3 = struct { .secret_key = secret_key, .endpoint_url = endpoint, .region = region, - }, .{}); + }); } pub fn deinit(self: *S3) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -455,7 +282,6 @@ pub const S3 = struct { if (self.config.secret_key) |secret_key| self.allocator.free(secret_key); self.allocator.free(self.config.endpoint_url); self.allocator.free(self.config.region); - self.allocator.free(self.config.auth_service); } pub fn io(self: *S3) std.Io { @@ -483,6 +309,13 @@ pub const S3 = struct { }; } + pub fn backend(self: *S3) Backend { + return .{ + .io = self.io(), + .read_hints = .{ .high_latency = true }, + }; + } + fn openHandle(self: *S3) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -532,9 +365,9 @@ pub const S3 = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -580,11 +413,9 @@ pub const S3 = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.BadPathName => return std.Io.File.OpenError.BadPathName, - else => return std.Io.File.OpenError.Unexpected, - }; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); return .{ .inode = 0, @@ -599,18 +430,19 @@ pub const S3 = struct { }; } - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + } fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.BadPathName => return std.Io.File.OpenError.BadPathName, - else => return std.Io.File.OpenError.Unexpected, - }; - var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -636,7 +468,16 @@ pub const S3 = struct { } const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + const objects = self.listObjects(handle.uri) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.OutOfMemory => return error.SystemResources, + // Logged by `exchange`. + error.RequestFailed => return error.Unexpected, + else => { + log.err("listing {s} failed: {t}", .{ handle.uri, err }); + return error.Unexpected; + }, + }; self.dir_read_states.put(self.allocator, reader, .{ .index = 0, @@ -721,10 +562,7 @@ pub const S3 = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { @@ -734,7 +572,7 @@ pub const S3 = struct { if (relative_offset >= 0) { handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); } else { - const abs_offset: u64 = @intCast(-relative_offset); + const abs_offset: u64 = @abs(relative_offset); if (abs_offset > handle.pos) { handle.pos = 0; } else { @@ -820,49 +658,37 @@ pub const S3 = struct { .fragment = null, }; - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); + var listing: Listing = .{ .s3 = self, .uri = uri }; + var target_buffer: [256]u8 = undefined; + return request.perform([]u8, &self.governor, .{ + .target = std.fmt.bufPrint(&target_buffer, "{s}/{s}", .{ bucket, key_prefix }) catch bucket, + .key = request.authorityOf(uri), + }, &listing, Listing.attempt); + } - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; + /// The bucket listing, signed again on every attempt like every other + /// S3 request (`Signature`). + const Listing = struct { + s3: *S3, + uri: std.Uri, + signature: Signature = .{}, - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); - - var req = try self.client.request(.GET, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [2 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to list object {f}", .{uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; + fn attempt(self: *Listing, current: request.Attempt) anyerror!request.Outcome([]u8) { + return request.exchange([]u8, current, self.uri, .{ + .authorization = try self.signature.sign(self.s3, .GET, self.uri, &.{}), + .extra_headers = &self.signature.headers, + }, self, Listing.consume); } - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(self.allocator, content_len) - else - // When we don't have a content length, put a reasonable limit - try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); - } + fn consume(self: *Listing, res: *std.http.Client.Response) anyerror![]u8 { + const allocator = self.s3.allocator; + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(allocator, content_len) + else + // When we don't have a content length, put a reasonable limit + try res.reader(&.{}).allocRemaining(allocator, .limited(1024 * 1024)); + } + }; fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); @@ -905,93 +731,132 @@ pub const S3 = struct { return try results.toOwnedSlice(self.allocator); } - fn fetchSize(self: *S3, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; + fn fetchSize(self: *S3, path: []const u8) !u64 { var url_buf: [8 * 1024]u8 = undefined; - - const path = try self.resolvePath(dir, sub_path, &path_buffer); const url = try self.s3Url(path, &url_buf); const uri = std.Uri.parse(url) catch return error.BadPathName; - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); - - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .HEAD, uri, timestamp, &.{}); - - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - const res = try req.receiveHead(&redirect_buffer); - - return switch (res.head.status.class()) { - .success => res.head.content_length.?, - else => switch (res.head.status) { - .not_found => return error.FileNotFound, - else => blk: { - log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); - break :blk error.ServerError; - }, - }, - }; + var head: SizeRequest = .{ .s3 = self, .uri = uri }; + return request.perform(u64, &self.governor, .{ .target = url, .key = request.authorityOf(uri) }, &head, SizeRequest.attempt); } - fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { - const read_size = parallel_read.readSize(handle.size, offset, data); - if (read_size == 0) return 0; + /// The object's size: one signed HEAD (`ExchangeOptions.open`; S3's 403 + /// for a missing key: vfs.zig's header). + const SizeRequest = struct { + s3: *S3, + uri: std.Uri, + signature: Signature = .{}, - var url_buf: [8 * 1024]u8 = undefined; - const url = try self.s3Url(handle.uri, &url_buf); - const uri = std.Uri.parse(url) catch return error.BadPathName; - - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .backend = self, - .uri = uri, - .url = url, - }; - - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, - }; + fn attempt(self: *SizeRequest, current: request.Attempt) anyerror!request.Outcome(u64) { + return request.exchange(u64, current, self.uri, .{ + .method = .HEAD, + .authorization = try self.signature.sign(self.s3, .HEAD, self.uri, &.{}), + .extra_headers = &self.signature.headers, + .open = true, + }, self, SizeRequest.consume); } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; + fn consume(_: *SizeRequest, res: *std.http.Client.Response) anyerror!u64 { + return res.head.content_length orelse error.MissingContentLength; + } + }; - return read_size; + fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { + var url_buf: [8 * 1024]u8 = undefined; + const url = try self.s3Url(handle.uri, &url_buf); + var signed: SignedRequest = .{ .s3 = self, .uri = std.Uri.parse(url) catch return error.BadPathName }; + return range_read.performRangeRead(&self.governor, .{ + .request = .{ .target = url, .key = request.authorityOf(signed.uri) }, + }, &signed, SignedRequest.prepare, data, offset, handle.size); } + + /// One SigV4-signed GET. The signature covers `x-amz-date` and `Range`, + /// so it is recomputed for every attempt. + const SignedRequest = struct { + s3: *S3, + uri: std.Uri, + signature: Signature = .{}, + + fn prepare(self: *SignedRequest, _: request.Attempt, range: std.http.Header) anyerror!range_read.PreparedRequest { + return .{ + .uri = self.uri, + .authorization = try self.signature.sign(self.s3, .GET, self.uri, &.{range}), + .extra_headers = &self.signature.headers, + }; + } + }; + + /// One attempt's SigV4 headers. Every request signs each attempt anew, + /// since the signature covers `x-amz-date`. The returned authorization + /// and `headers` point into this storage. + const Signature = struct { + timestamp: [16]u8 = undefined, + authorization: [512]u8 = undefined, + headers: [2]std.http.Header = undefined, + + /// The authorization over `uri` and `signed_headers` beside the two + /// headers it sets; omitted without credentials. + fn sign( + self: *Signature, + s3: *S3, + method: std.http.Method, + uri: std.Uri, + signed_headers: []const std.http.Header, + ) !std.http.Client.Request.Headers.Value { + const timestamp = try s3.getTimestamp(&self.timestamp); + const signer: AwsSigV4 = .{ + .access_key = s3.config.access_key, + .secret_key = s3.config.secret_key, + .region = s3.config.region, + .service = auth_service, + }; + const authorization = try signer.generateAuthHeader(&self.authorization, method, uri, timestamp, signed_headers); + self.headers = .{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }; + return if (authorization) |value| .{ .override = value } else .omit; + } + }; }; + +test "AwsSigV4 signs the HEAD, listing and Range GET that S3 sends" { + // AWS's example keys and date. The expected signatures are botocore's + // (S3SigV4Auth) for the same requests, with UNSIGNED-PAYLOAD as every + // request here is sent. + const signer: AwsSigV4 = .{ + .access_key = "AKIAIOSFODNN7EXAMPLE", + .secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + .region = "us-east-1", + .service = "s3", + }; + const timestamp = "20130524T000000Z"; + const credential = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, "; + var buffer: [512]u8 = undefined; + + const object = try std.Uri.parse("https://s3.amazonaws.com/examplebucket/test.txt"); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=d360d9545e619df15fd61e08c5b21c3e2219ee708af3b95ac439b0ee6fdf44b0", + (try signer.generateAuthHeader(&buffer, .HEAD, object, timestamp, &.{})).?, + ); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f483aaef7a23c6af727aeed791410b7f43f0d696542829b6bf6edc1c7cb112c4", + (try signer.generateAuthHeader(&buffer, .GET, object, timestamp, &.{.{ .name = "Range", .value = "bytes=0-9" }})).?, + ); + const listing = try std.Uri.parse("https://s3.amazonaws.com/examplebucket?delimiter=%2F&list-type=2&max-keys=1000&prefix=dir%2F"); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0a8491caa0b04a231c4ea74a87bde51ef60fbe69b496b206f4d07a4c9031188a", + (try signer.generateAuthHeader(&buffer, .GET, listing, timestamp, &.{})).?, + ); + // A custom endpoint signs its port as part of the host. + const minio = try std.Uri.parse("http://127.0.0.1:9000/models/dir/obj.bin"); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=84ff2d687bd88454372c3277af863c28ca047204936b346b80bf93456d0870d4", + (try signer.generateAuthHeader(&buffer, .HEAD, minio, timestamp, &.{})).?, + ); + + // Without both keys, requests go unsigned. + const anonymous: AwsSigV4 = .{ .access_key = "AKIAIOSFODNN7EXAMPLE", .secret_key = null, .region = "us-east-1", .service = "s3" }; + try std.testing.expectEqual(null, try anonymous.generateAuthHeader(&buffer, .HEAD, object, timestamp, &.{})); +} diff --git a/vfs/vfs.zig b/vfs/vfs.zig index fe626ff9..8a52e763 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -1,25 +1,87 @@ +//! A `std.Io` over local files and remote objects. A bare path or a +//! `file://` URI is served by the inner io; a URI such as +//! `hf://org/model/model.safetensors` or `s3://bucket/key` by the backend +//! registered for its scheme (`registerBackend`, or the four of `Remotes`). +//! Readers take `io()` and need not know which they read; one that must +//! (the loader, for read sizes and direct I/O) asks `fromIo`. +//! +//! The VFS, and every backend registered on it, must outlive each file +//! opened through it and each reader that keeps one open: a `Loader` may +//! keep its source files open until its `deinit`. +//! +//! Errors of a remote path, alike on `HTTP`, `HF`, `S3` and `GCS`: opening, +//! stat-ing or probing it (`access`) fails with `FileNotFound` for a missing +//! object (404, 410, or a path absent from an `HF` repository's listing), +//! `PermissionDenied` for a denied one (401, 403), `Canceled` for a canceled +//! task, and `SystemResources` when a request cannot allocate. Any other +//! failure of a request is logged at err with the backend and the path, and +//! surfaces as `Unexpected`. On S3 a principal without `s3:ListBucket` gets +//! 403 for a missing key too, so a probe of an optional object treats both +//! as absent. A positional read that still fails after its retries is +//! logged at err and surfaces as `Unexpected`; a streaming read as +//! `InputOutput`. A throttled backend holds its requests rather than failing +//! them, until its throttle budget runs out (`request.Retry`). A remote path +//! is read-only: creating or deleting a file or a directory there, or +//! opening a file for writing, is logged at err and fails with +//! `ReadOnlyFileSystem`. A local file fails as the inner io does (a denied +//! one is `AccessDenied` there); it may be read past the page cache +//! (`useDirectIo`). Each remote backend logs its read totals at debug from +//! its `deinit`. const std = @import("std"); const stdx = @import("stdx"); -pub const File = @import("file.zig").File; pub const GCS = @import("gcs.zig").GCS; pub const HF = @import("hf.zig").HF; pub const HTTP = @import("http.zig").HTTP; pub const S3 = @import("s3.zig").S3; -pub const VFSBase = @import("base.zig").VFSBase; +/// A test-only HTTP origin for tests above this layer (mock_server.zig). +pub const MockServer = @import("mock_server.zig").MockServer; +const base_module = @import("base.zig"); +const direct_io = @import("direct_io.zig"); +pub const Backend = base_module.Backend; +pub const ReadHints = base_module.ReadHints; +const VFSBase = base_module.VFSBase; +/// See `useDirectIo`. +pub const DirectIo = direct_io.Policy; +/// What every read of a direct file must meet in offset, buffers and total +/// (`useDirectIo`); 0 on a system without direct I/O, where no file is. +pub const direct_io_alignment: usize = if (direct_io.supported) direct_io.alignment else 0; + +test { + _ = @import("gcs.zig"); + _ = @import("http_acceptance_test.zig"); +} const log = std.log.scoped(.@"zml/vfs"); +/// For the refusals of writes to a remote path, which tests trigger on +/// purpose; every other err here still fails a test (`scopedLog`). +const refusal_log = base_module.scopedLog(.@"zml/vfs"); -const CWD_HANDLE: u32 = 0; +/// Indices 0-2 are the standard streams, so `std.Io.File.stdout()` and +/// friends resolve to their own descriptor; the cwd placeholder follows. +const CWD_HANDLE: u32 = 3; +/// Local files are the VFS's own: a bare path or a `file://` URI resolves +/// to the inner `Io`, never to a registered backend. +const local_scheme = "file"; const VFS = @This(); -const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; +const Handle = struct { + handle: u32, + backend_idx: ?usize, + flags: std.Io.File.Flags = .{ .nonblocking = false }, + /// A local file's read mode (`useDirectIo`). + direct: direct_io.Mode = .{}, + + fn innerFile(self: *const Handle) std.Io.File { + return .{ .handle = @intCast(self.handle), .flags = self.flags }; + } +}; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, -backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, +backends: std.StringArrayHashMapUnmanaged(Backend) = .empty, handles: stdx.SegmentedList(Handle, 128) = .{}, closed_handles: std.ArrayList(u32) = .empty, @@ -29,10 +91,11 @@ pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { const base = VFSBase.init(base_io); var handles: @FieldType(@This(), "handles") = .{}; - try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); + std.debug.assert(handles.len == CWD_HANDLE + 1); return .{ .allocator = allocator, @@ -47,49 +110,135 @@ pub fn deinit(self: *VFS) void { self.backends.deinit(self.allocator); } -pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { +/// Serves `scheme` with `backend`, replacing any backend registered under +/// it. `scheme` is kept, not copied: it must outlive the VFS. +pub fn registerBackend(self: *VFS, scheme: []const u8, backend: Backend) std.mem.Allocator.Error!void { + std.debug.assert(!std.mem.eql(u8, scheme, local_scheme)); // file is the VFS's own (local_scheme) self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); try self.backends.put(self.allocator, scheme, backend); } -pub fn unregister(self: *VFS, scheme: []const u8) bool { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); +/// The remote schemes on one HTTP client: `https`, `hf`, `s3` and `gs`, each +/// configured from the environment through its backend's `auto` (plain +/// `HTTP.init` for `https`), the client honoring the proxy variables. +/// Initialized in place, since the backends point at the client; it must not +/// move afterwards. Close every file opened through these schemes before +/// `deinit` (deinitialize a `Loader` first); the VFS must not reach them +/// after it. Their errors follow this module's contract. +pub const Remotes = struct { + /// The proxies' allocations, which must outlive the client. + arena: std.heap.ArenaAllocator, + client: std.http.Client, + https: HTTP, + hf: HF, + s3: S3, + gs: GCS, - return self.backends.remove(scheme); -} + /// Builds the client on the VFS's base io and registers the four + /// schemes, replacing any backend registered under them. `environ` is + /// read here only: tokens, credentials and proxies are copied. On error + /// nothing is registered. + pub fn init(self: *Remotes, allocator: std.mem.Allocator, vfs: *VFS, environ: *std.process.Environ.Map) !void { + const base_io = vfs.base.inner; + self.arena = .init(allocator); + errdefer self.arena.deinit(); + self.client = .{ .allocator = allocator, .io = base_io }; + errdefer self.client.deinit(); + try self.client.initDefaultProxies(self.arena.allocator(), environ); + self.https = try .init(allocator, base_io, &self.client, .{ .protocol = .https }); + errdefer self.https.deinit(); + self.hf = try .auto(allocator, base_io, &self.client, environ); + errdefer self.hf.deinit(); + self.s3 = try .auto(allocator, base_io, &self.client, environ); + errdefer self.s3.deinit(); + self.gs = try .auto(allocator, base_io, &self.client, environ); + errdefer self.gs.deinit(); + vfs.mutex.lockUncancelable(base_io); + defer vfs.mutex.unlock(base_io); + // Room first: once a scheme is registered, nothing can fail. + try vfs.backends.ensureUnusedCapacity(vfs.allocator, 4); + vfs.backends.putAssumeCapacity("https", self.https.backend()); + vfs.backends.putAssumeCapacity("hf", self.hf.backend()); + vfs.backends.putAssumeCapacity("s3", self.s3.backend()); + vfs.backends.putAssumeCapacity("gs", self.gs.backend()); + } + + pub fn deinit(self: *Remotes) void { + self.gs.deinit(); + self.s3.deinit(); + self.hf.deinit(); + self.https.deinit(); + self.client.deinit(); + self.arena.deinit(); + } +}; + +/// The `std.Io` of this VFS. It points into the VFS, which must not move +/// once this was called. pub fn io(self: *VFS) std.Io { return .{ .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), + .vtable = &io_vtable, }; } +/// The VFS behind an `Io`, or null for any other implementation: a reader +/// that opened a file through `any_io` asks this VFS about that handle. +pub fn fromIo(any_io: std.Io) ?*VFS { + if (any_io.vtable != &io_vtable) return null; + return @fieldParentPtr("base", VFSBase.as(any_io.userdata)); +} + +const io_vtable: std.Io.VTable = VFSBase.vtable(.{ + .operate = operate, + .dirCreateDir = dirCreateDir, + .dirCreateDirPath = dirCreateDirPath, + .dirCreateDirPathOpen = dirCreateDirPathOpen, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirCreateFileAtomic = dirCreateFileAtomic, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .dirDeleteFile = dirDeleteFile, + .dirDeleteDir = dirDeleteDir, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, +}); + +/// The read hints of the backend serving `path`, for a reader sizing its +/// requests. A path resolves as it does for `openFile`: a bare path or a +/// `file://` URI is local (`ReadHints.local`), and a scheme no backend +/// serves is `VFSNotRegistered`. +pub fn readHints(self: *VFS, path: []const u8) !ReadHints { + const backend_idx, _, _ = try self.lookupDir(.cwd(), path); + const index = backend_idx orelse return .local; + return self.backends.entries.items(.value)[index].read_hints; +} + +/// Whether `path` resolves to a local file, the only kind `useDirectIo` can +/// read directly; an unregistered scheme is not. Opens nothing. +pub fn isLocal(self: *VFS, path: []const u8) bool { + const backend_idx, _, _ = self.lookupDir(.cwd(), path) catch return false; + return backend_idx == null; +} + fn openHandle(self: *VFS) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -117,6 +266,53 @@ fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { return .{ handle, self.getBackend(handle.backend_idx) }; } +/// Direct reads for a local file under `policy`: sets the flag on a +/// read-only descriptor of a regular file at least one alignment unit long +/// (and, under `auto`, mostly out of the page cache) and returns true, after +/// which every read of the file must meet `direct_io_alignment` in offset, +/// buffers and total. False leaves the file buffered: another backend's +/// file, `off`, a cached file under `auto`, a filesystem that refuses the +/// flag, or a file whose direct read the kernel already rejected. The first +/// call under `on` or `auto` decides for the handle's life +/// (`direct_io.Mode`); the caller must ask before it reads. The mechanism +/// lives in the VFS because it owns the descriptor; one descriptor suffices +/// because a reader reads a file in one mode. +pub fn useDirectIo(self: *VFS, file: std.Io.File, policy: DirectIo) bool { + const handle, _ = self.getFileHandle(file); + if (handle.backend_idx != null) return false; + return handle.direct.decide(self.base.inner, handle.innerFile(), policy); +} + +/// See `predictDirectIo`. +pub const DirectIoPrediction = direct_io.Prediction; + +/// What `useDirectIo` would answer under `policy` for the local file +/// `sub_path` of `dir`, and its size, asked on a descriptor of its own that +/// is closed again: no handle is decided and nothing is logged. For a reader +/// that places memory before it plans its files. Under `auto` this sample +/// caches what it misses, so a decision that follows may find any file +/// cached and read it buffered, the likelier the fewer pages it has per +/// sample (`direct_io.predict`). Null for a path that is not local, or a +/// file that does not open or stat. +pub fn predictDirectIo(self: *VFS, dir: std.Io.Dir, sub_path: []const u8, policy: DirectIo) error{Canceled}!?DirectIoPrediction { + const backend_idx, const inner_dir, const inner = self.lookupDir(dir, sub_path) catch return null; + if (backend_idx != null) return null; + const file = inner_dir.openFile(inner, stripScheme(sub_path), .{ .mode = .read_only }) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return null, + }; + defer file.close(inner); + return direct_io.predict(inner, file, policy); +} + +/// A local file as the inner `Io` may read it, off direct reads +/// (`direct_io.Mode.leave`). +fn innerReadable(self: *VFS, handle: *Handle) std.Io.File { + const file = handle.innerFile(); + handle.direct.leave(self.base.inner, file); + return file; +} + fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -130,7 +326,7 @@ fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { } fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { - if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; + if (backend_idx) |idx| return self.backends.entries.items(.value)[idx].io else return self.base.inner; } fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { @@ -141,6 +337,7 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz if (sub_path) |sp| { if (std.mem.indexOf(u8, sp, "://") != null) { const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; + if (std.mem.eql(u8, uri.scheme, local_scheme)) return .{ null, std.Io.Dir.cwd(), self.base.inner }; const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { if (std.mem.eql(u8, uri.scheme, s)) break idx; } else return error.VFSNotRegistered; @@ -149,19 +346,8 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz } if (std.meta.eql(dir, std.Io.Dir.cwd())) { - if (sub_path == null) return .{ null, dir, self.base.inner }; - if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; - - const uri = std.Uri.parse(sub_path.?) catch null; - if (uri) |u| { - const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { - if (std.mem.eql(u8, u.scheme, s)) break idx; - } else null; - if (backend_idx == null) return error.VFSNotRegistered; - return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; - } else { - return .{ null, std.Io.Dir.cwd(), self.base.inner }; - } + // Without "://", a path from cwd is local, a colon in it included. + return .{ null, dir, self.base.inner }; } else { const handle = self.getDirHandle(dir); if (handle.backend_idx) |backend_idx| { @@ -172,9 +358,26 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz } } +/// `path` without its "scheme://" prefix; a name with a colon and no "://" +/// is a plain local name. fn stripScheme(path: []const u8) []const u8 { - const uri = std.Uri.parse(path) catch return path; - return path[uri.scheme.len + 3 ..]; + const i = std.mem.indexOf(u8, path, "://") orelse return path; + return path[i + 3 ..]; +} + +/// The inner `Io`'s directory for creating or deleting `sub_path` from +/// `dir`. A remote path is refused: no remote backend writes, and the base +/// layer under each one would write a local file in its place. +fn writableDir(self: *VFS, dir: std.Io.Dir, sub_path: []const u8, operation: []const u8) error{ ReadOnlyFileSystem, Unexpected }!std.Io.Dir { + const backend_idx, const dir_, _ = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend to {s} '{s}' : {any}", .{ operation, sub_path, err }); + return error.Unexpected; + }; + if (self.getScheme(backend_idx)) |scheme| { + refusal_log.err("Cannot {s} '{s}': {s} paths are read-only", .{ operation, sub_path, scheme }); + return error.ReadOnlyFileSystem; + } + return dir_; } fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { @@ -183,7 +386,7 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable .file_read_streaming => |o| { const handle, const backend = self.getFileHandle(o.file); return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ - .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .file = self.innerReadable(handle), .data = o.data, } }); }, @@ -202,6 +405,32 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable } } +fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create directory"); + return self.base.inner.vtable.dirCreateDir(self.base.inner.userdata, dir_, stripScheme(sub_path), permissions); +} + +fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create directory"); + return self.base.inner.vtable.dirCreateDirPath(self.base.inner.userdata, dir_, stripScheme(sub_path), permissions); +} + +fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create directory"); + const inner = self.base.inner; + const fs_dir = try inner.vtable.dirCreateDirPathOpen(inner.userdata, dir_, stripScheme(sub_path), permissions, options); + errdefer fs_dir.close(inner); + const idx, const handle = self.openHandle() catch return std.Io.Dir.CreateDirPathOpenError.Unexpected; + handle.* = .{ + .handle = @intCast(fs_dir.handle), + .backend_idx = null, + }; + return .{ .handle = @intCast(idx) }; +} + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { @@ -246,28 +475,36 @@ pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, o pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); - return std.Io.File.OpenError.Unexpected; - }; - - const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); - errdefer file.close(backend); + const dir_ = try self.writableDir(dir, sub_path, "create file"); + const inner = self.base.inner; + const file = try inner.vtable.dirCreateFile(inner.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(inner); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), - .backend_idx = backend_idx, + .backend_idx = null, .flags = file.flags, }; return .{ .handle = @intCast(idx), .flags = file.flags }; } +/// The atomic file holds the inner `Io`'s handles, not this VFS's. +fn dirCreateFileAtomic(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.CreateFileAtomicOptions) std.Io.Dir.CreateFileAtomicError!std.Io.File.Atomic { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create file"); + return self.base.inner.vtable.dirCreateFileAtomic(self.base.inner.userdata, dir_, stripScheme(sub_path), options); +} + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); return std.Io.File.OpenError.Unexpected; }; + if (flags.mode != .read_only and backend_idx != null) { + refusal_log.err("Cannot open '{s}' for writing: {s} paths are read-only", .{ sub_path, self.getScheme(backend_idx).? }); + return error.ReadOnlyFileSystem; + } const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; @@ -330,13 +567,25 @@ fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8 if (self.getScheme(backend_idx)) |s| { const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, stripScheme(path_name), out_buffer[prefix.len..]); return prefix.len + path_len; } else { - return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); + return try backend.vtable.dirRealPathFile(backend.userdata, dir_, stripScheme(path_name), out_buffer); } } +fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "delete file"); + return self.base.inner.vtable.dirDeleteFile(self.base.inner.userdata, dir_, stripScheme(sub_path)); +} + +fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "delete directory"); + return self.base.inner.vtable.dirDeleteDir(self.base.inner.userdata, dir_, stripScheme(sub_path)); +} + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); @@ -376,7 +625,7 @@ fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []co const original_src_io = reader.io; const original_src_handle = reader.file; reader.io = src_backend; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + reader.file = self.innerReadable(src_handle); defer { reader.io = original_src_io; reader.file = original_src_handle; @@ -397,7 +646,7 @@ fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []c const original_src_io = reader.io; const original_src_handle = reader.file; reader.io = src_backend; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + reader.file = self.innerReadable(src_handle); defer { reader.io = original_src_io; reader.file = original_src_handle; @@ -409,6 +658,7 @@ fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []c fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); + if (try handle.direct.read(self.base.inner, handle.innerFile(), data, offset)) |bytes| return bytes; return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); } @@ -437,6 +687,281 @@ fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std. } } +test "VFS gives the read hints of local and registered paths" { + var filesystem = try VFS.init(std.testing.allocator, std.testing.io); + defer filesystem.deinit(); + const hints: ReadHints = .{ .read_chunk_size = 32 * 1024 * 1024, .high_latency = true }; + try filesystem.registerBackend("test", .{ .io = std.testing.io, .read_hints = hints }); + + try std.testing.expectEqual(hints, try filesystem.readHints("test://bucket/object")); + try std.testing.expect(!filesystem.isLocal("test://bucket/object")); + + for ([_][]const u8{ "/tmp/model.safetensors", "models/model.safetensors", "file:///tmp/model.safetensors" }) |path| { + try std.testing.expectEqual(ReadHints{ .read_chunk_size = 8 * 1024 * 1024, .high_latency = false }, try filesystem.readHints(path)); + try std.testing.expect(filesystem.isLocal(path)); + } + try std.testing.expectError(error.VFSNotRegistered, filesystem.readHints("missing://bucket/object")); + try std.testing.expect(!filesystem.isLocal("missing://bucket/object")); + try std.testing.expectEqual(@as(?*VFS, &filesystem), VFS.fromIo(filesystem.io())); + try std.testing.expectEqual(@as(?*VFS, null), VFS.fromIo(std.testing.io)); +} + +test "Remotes registers every remote scheme with its read hints" { + var filesystem = try VFS.init(std.testing.allocator, std.testing.io); + defer filesystem.deinit(); + // No credentials and no proxies: nothing is read from the user's home. + var environ: std.process.Environ.Map = .init(std.testing.allocator); + defer environ.deinit(); + var remotes: Remotes = undefined; + try remotes.init(std.testing.allocator, &filesystem, &environ); + defer remotes.deinit(); + + const Case = struct { path: []const u8, local: bool = false, hints: ReadHints }; + const remote: ReadHints = .{ .read_chunk_size = 16 * 1024 * 1024, .high_latency = true }; + const cases = [_]Case{ + .{ .path = "file:///tmp/model", .local = true, .hints = .local }, + .{ .path = "/var/models/model", .local = true, .hints = .local }, + .{ .path = "https://example.com/model", .hints = remote }, + .{ .path = "s3://bucket/model", .hints = remote }, + .{ .path = "gs://bucket/model", .hints = remote }, + .{ .path = "hf://owner/model", .hints = .{ .read_chunk_size = 32 * 1024 * 1024, .high_latency = true } }, + }; + for (cases) |case| { + try std.testing.expectEqual(case.hints, try filesystem.readHints(case.path)); + // Only a local file can be read directly; nothing is opened to ask. + try std.testing.expectEqual(case.local, filesystem.isLocal(case.path)); + if (case.local) continue; + // A remote real path carries its scheme once, resolved without a request. + var real_path: [64]u8 = undefined; + try std.testing.expectEqualStrings(case.path, real_path[0..try std.Io.Dir.cwd().realPathFile(filesystem.io(), case.path, &real_path)]); + } +} + +test "VFS serves bare paths and file:// URIs from the inner Io" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const created = try tmp.dir.createFile(testing_io, "bare.bin", .{ .read = true }); + try created.writePositionalAll(testing_io, "bare path bytes", 0); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; + created.close(testing_io); + + var filesystem = try VFS.init(std.testing.allocator, testing_io); + defer filesystem.deinit(); + const vfs_io = filesystem.io(); + + var uri_buffer: [std.fs.max_path_bytes + 8]u8 = undefined; + const uri = try std.fmt.bufPrint(&uri_buffer, "file://{s}", .{path}); + for ([_][]const u8{ path, uri }) |open_path| { + var real_buffer: [std.fs.max_path_bytes]u8 = undefined; + try std.testing.expectEqualStrings(path, real_buffer[0..try std.Io.Dir.cwd().realPathFile(vfs_io, open_path, &real_buffer)]); + const opened = try std.Io.Dir.openFile(.cwd(), vfs_io, open_path, .{ .mode = .read_only }); + defer opened.close(vfs_io); + const handle, _ = filesystem.getFileHandle(opened); + try std.testing.expectEqual(@as(?usize, null), handle.backend_idx); + // A file this small is never direct. + try std.testing.expect(!filesystem.useDirectIo(opened, .on)); + var contents: [15]u8 = undefined; + try std.testing.expectEqual(contents.len, try opened.readPositionalAll(vfs_io, &contents, 0)); + try std.testing.expectEqualStrings("bare path bytes", &contents); + } +} + +test "VFS reads the same bytes under every direct I/O policy" { + const testing_io = std.testing.io; + const alignment = 4096; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var contents: [3 * alignment + 100]u8 = undefined; + for (&contents, 0..) |*byte, i| byte.* = @truncate(i * 7 + 3); + const created = try tmp.dir.createFile(testing_io, "data.bin", .{ .read = true }); + try created.writePositionalAll(testing_io, &contents, 0); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; + created.close(testing_io); + + var filesystem = try VFS.init(std.testing.allocator, testing_io); + defer filesystem.deinit(); + const vfs_io = filesystem.io(); + const buffer = try std.testing.allocator.alignedAlloc(u8, .fromByteUnits(alignment), 2 * alignment); + defer std.testing.allocator.free(buffer); + const plain: []u8 = buffer; + for ([_]DirectIo{ .on, .auto, .off }) |policy| { + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); + defer file.close(vfs_io); + // Direct when the policy and the filesystem allow it, buffered + // otherwise: `off`, or under `auto` a file just written and thus + // cached. The bytes are the same either way. + const direct = filesystem.useDirectIo(file, policy); + if (policy != .on) try std.testing.expect(!direct); + // The decision stands; `off` neither asks nor changes it. + try std.testing.expect(!filesystem.useDirectIo(file, .off)); + try std.testing.expectEqual(direct, filesystem.useDirectIo(file, policy)); + + // Aligned offset, buffer and length. + @memset(buffer, 0); + try std.testing.expectEqual(buffer.len, try file.readPositionalAll(vfs_io, buffer, alignment)); + try std.testing.expectEqualSlices(u8, contents[alignment..][0..buffer.len], buffer); + + // One aligned call that runs past the end of the file is cut there. + @memset(buffer, 0); + const past_end = try file.readPositional(vfs_io, &.{plain}, 2 * alignment); + try std.testing.expectEqual(contents.len - 2 * alignment, past_end); + try std.testing.expectEqualSlices(u8, contents[2 * alignment ..], buffer[0..past_end]); + try std.testing.expectEqual(direct, filesystem.useDirectIo(file, policy)); + + // A streaming read takes the file off direct reads for good. (A + // misaligned positional read of a direct file does too, with an + // error logged, which the test runner would count; a filesystem + // such as tmpfs may serve it instead, so it is not exercised here.) + var head: [8]u8 = undefined; + const head_slice: []u8 = &head; + try std.testing.expectEqual(head.len, try file.readStreaming(vfs_io, &.{head_slice})); + try std.testing.expectEqualSlices(u8, contents[0..head.len], &head); + try std.testing.expect(!filesystem.useDirectIo(file, policy)); + + // Buffered from now on: unaligned reads and the end of the file. + var tail: [50]u8 = undefined; + try std.testing.expectEqual(tail.len, try file.readPositionalAll(vfs_io, &tail, alignment + 3)); + try std.testing.expectEqualSlices(u8, contents[alignment + 3 ..][0..tail.len], &tail); + try std.testing.expectEqual(@as(usize, 0), try file.readPositional(vfs_io, &.{plain}, contents.len)); + } + + // A descriptor opened for writing is never direct: the flag would + // constrain its writes too. + const writable = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_write }); + defer writable.close(vfs_io); + try std.testing.expect(!filesystem.useDirectIo(writable, .on)); + const writable_handle, _ = filesystem.getFileHandle(writable); + if (comptime direct_io.supported) try std.testing.expectEqual(std.os.linux.E.BADF, direct_io.enable(writable_handle.innerFile().handle)); +} + +test "predictDirectIo answers as useDirectIo would and decides no handle" { + if (comptime !direct_io.supported) return error.SkipZigTest; + const testing_io = std.testing.io; + const alignment = 4096; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents: [3 * alignment + 100]u8 = @splat(0x5a); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "data.bin", .data = &contents }); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "short.bin", .data = contents[0..100] }); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(testing_io, &root_buffer)]; + + var filesystem = try VFS.init(std.testing.allocator, testing_io); + defer filesystem.deinit(); + // Another backend's scheme, served here by the plain io: its paths open, + // but they are not local. + try filesystem.registerBackend("remote", .{ .io = testing_io }); + const vfs_io = filesystem.io(); + const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); + defer dir.close(vfs_io); + var path_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, "{s}/data.bin", .{root}); + + for ([_]DirectIo{ .on, .auto, .off }) |policy| { + // No handle is opened or decided to answer. + const handles = filesystem.handles.len; + const closed_handles = filesystem.closed_handles.items.len; + const predicted = (try filesystem.predictDirectIo(.cwd(), path, policy)).?; + try std.testing.expectEqual(@as(u64, contents.len), predicted.size); + try std.testing.expectEqual(predicted, (try filesystem.predictDirectIo(dir, "data.bin", policy)).?); + try std.testing.expectEqual(handles, filesystem.handles.len); + try std.testing.expectEqual(closed_handles, filesystem.closed_handles.items.len); + + // A fresh handle decides as predicted: direct only under `on`, where + // the filesystem allows it, since the file was just written and so + // is cached. + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); + defer file.close(vfs_io); + try std.testing.expectEqual(predicted.direct, filesystem.useDirectIo(file, policy)); + if (policy != .on) try std.testing.expect(!predicted.direct); + } + + // A file shorter than a unit is never direct. + const short = (try filesystem.predictDirectIo(dir, "short.bin", .on)).?; + try std.testing.expectEqual(DirectIoPrediction{ .size = 100, .direct = false }, short); + // Nothing to answer for a remote path, though its backend opens the same + // file, or for a missing file. + var remote_buffer: [std.fs.max_path_bytes + 32]u8 = undefined; + const remote = try std.fmt.bufPrint(&remote_buffer, "remote://{s}/data.bin", .{root}); + try std.testing.expectEqual(null, try filesystem.predictDirectIo(.cwd(), remote, .on)); + try std.testing.expectEqual(null, try filesystem.predictDirectIo(dir, "missing.bin", .on)); +} + +test "auto keeps reading a cold file direct after probing it" { + if (comptime !direct_io.supported) return error.SkipZigTest; + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // One 2 MiB window per sample (`direct_io.residency_samples`): many + // pages each, on 64 KiB-page kernels too. + const window = 2 << 20; + const contents = try std.testing.allocator.alloc(u8, direct_io.residency_samples * window); + defer std.testing.allocator.free(contents); + @memset(contents, 0x5a); + const created = try tmp.dir.createFile(testing_io, "cold.bin", .{ .read = true }); + try created.writePositionalAll(testing_io, contents, 0); + try created.sync(testing_io); + _ = std.os.linux.fadvise(created.handle, 0, 0, std.os.linux.POSIX_FADV.DONTNEED); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; + created.close(testing_io); + + // The testing io, but the probes draw shifts a quarter window apart, + // 512 KiB from each other's samples. At random shifts a probe may sample + // the pages an earlier one's misses read, and find the file cached + // (`direct_io.predict`): about once in window / page pairs of probes. + const Shifts = struct { + var drawn: u64 = 0; + + fn random(_: ?*anyopaque, buffer: []u8) void { + std.mem.writeInt(u64, buffer[0..8], window / 8 + drawn * window / 4, .native); + drawn += 1; + } + }; + Shifts.drawn = 0; + var vtable = testing_io.vtable.*; + vtable.random = Shifts.random; + const shifting_io: std.Io = .{ .userdata = testing_io.userdata, .vtable = &vtable }; + + var filesystem = try VFS.init(std.testing.allocator, shifting_io); + defer filesystem.deinit(); + const vfs_io = filesystem.io(); + for (0..2) |open| { + // Predicted first, as a reader placing memory before it plans. + const predicted = (try filesystem.predictDirectIo(.cwd(), path, .auto)).?; + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); + defer file.close(vfs_io); + const direct = filesystem.useDirectIo(file, .auto); + // A filesystem that kept the file cached (tmpfs) or refused direct + // reads says nothing here. + if (open == 0 and !direct) return error.SkipZigTest; + try std.testing.expect(predicted.direct); + // The first probes' misses queued reads of the pages they sampled; + // once they landed, sampling the same pages would find the file + // cached. + if (open == 0) try testing_io.sleep(.fromMilliseconds(250), .awake); + try std.testing.expect(direct); + } + // Each probe drew a shift of its own. + try std.testing.expectEqual(@as(u64, 4), Shifts.drawn); +} + +test "standard streams resolve to their own descriptors" { + var vfs: VFS = try .init(std.testing.allocator, std.testing.io); + defer vfs.deinit(); + + const files = [_]std.Io.File{ .stdin(), .stdout(), .stderr() }; + const fds = [_]std.posix.fd_t{ std.posix.STDIN_FILENO, std.posix.STDOUT_FILENO, std.posix.STDERR_FILENO }; + for (files, fds) |file, fd| { + const handle, const backend = vfs.getFileHandle(file); + try std.testing.expectEqual(fd, handle.innerFile().handle); + try std.testing.expect(backend.vtable == vfs.base.inner.vtable); + } +} + test "copy a local file through the VFS" { const allocator = std.testing.allocator; const process_io = std.testing.io; @@ -457,11 +982,9 @@ test "copy a local file through the VFS" { const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); defer allocator.free(destination_uri); - var file_backend: File = .init(allocator, process_io, .{}); - defer file_backend.deinit(); + // `file://` is the VFS's own scheme: no backend to register. var vfs: VFS = try .init(allocator, process_io); defer vfs.deinit(); - try vfs.register("file", file_backend.io()); const vfs_io = vfs.io(); { @@ -481,3 +1004,107 @@ test "copy a local file through the VFS" { defer allocator.free(copied); try std.testing.expectEqualSlices(u8, &source_data, copied); } + +test "creates and deletes on a local path go to the inner Io, from a path or an open directory" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; + var vfs: VFS = try .init(std.testing.allocator, testing_io); + defer vfs.deinit(); + const vfs_io = vfs.io(); + + var uri_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + { + const opened = try std.Io.Dir.cwd().createDirPathOpen(vfs_io, try std.fmt.bufPrint(&uri_buffer, "file://{s}/a/b", .{root}), .{}); + defer opened.close(vfs_io); + // Relative to the descriptor the VFS's handle stands for. + try opened.createDir(vfs_io, "c", .default_dir); + try opened.writeFile(vfs_io, .{ .sub_path = "c/file", .data = "bytes" }); + try tmp.dir.access(testing_io, "a/b/c/file", .{}); + try opened.deleteFile(vfs_io, "c/file"); + try opened.deleteDir(vfs_io, "c"); + // A colon without "://" is part of a local name. + try opened.writeFile(vfs_io, .{ .sub_path = "notes:old.txt", .data = "bytes" }); + try tmp.dir.access(testing_io, "a/b/notes:old.txt", .{}); + try opened.deleteFile(vfs_io, "notes:old.txt"); + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/b/notes:old.txt", .{})); + } + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/b/c", .{})); + try std.Io.Dir.cwd().createDirPath(vfs_io, try std.fmt.bufPrint(&uri_buffer, "{s}/a/d", .{root})); + try tmp.dir.access(testing_io, "a/d", .{}); + try std.Io.Dir.cwd().deleteDir(vfs_io, try std.fmt.bufPrint(&uri_buffer, "{s}/a/d", .{root})); + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/d", .{})); +} + +test "creates and deletes on a remote path fail and write nothing anywhere" { + const allocator = std.testing.allocator; + const testing_io = std.testing.io; + var object: [16]u8 = @splat(0x2a); + var server = try MockServer.init(testing_io, &object, .{}); + defer server.deinit(testing_io); + try server.start(testing_io); + + var client: std.http.Client = .{ .allocator = allocator, .io = testing_io }; + defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var s3 = try S3.init(allocator, testing_io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1" }); + defer s3.deinit(); + var vfs: VFS = try .init(allocator, testing_io); + defer vfs.deinit(); + try vfs.registerBackend("s3", s3.backend()); + const vfs_io = vfs.io(); + const cwd = std.Io.Dir.cwd(); + + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFile(vfs_io, "s3://bucket/key", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.openFile(vfs_io, "s3://bucket/key", .{ .mode = .read_write })); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFileAtomic(vfs_io, "s3://bucket/key", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDir(vfs_io, "s3://bucket/dir", .default_dir)); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPath(vfs_io, "s3://bucket/a/b")); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPathOpen(vfs_io, "s3://bucket/a/b", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteFile(vfs_io, "s3://bucket/key")); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteDir(vfs_io, "s3://bucket/dir")); + // A directory opened on the remote is no way around it. + const bucket = try cwd.openDir(vfs_io, "s3://bucket", .{}); + defer bucket.close(vfs_io); + try std.testing.expectError(error.ReadOnlyFileSystem, bucket.createFile(vfs_io, "key", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, bucket.deleteFile(vfs_io, "key")); + + // Nothing on local disk, where the base layer would have written, and + // no request to the server. + for ([_][]const u8{ "s3:", "bucket" }) |name| { + try std.testing.expectError(error.FileNotFound, cwd.access(testing_io, name, .{})); + } + try server.stop(testing_io); + try std.testing.expectEqual(0, server.head_requests.load(.acquire) + server.get_requests.load(.acquire)); +} + +test "a name with a colon and no \"://\" is local" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "notes:old.txt", .data = "bytes" }); + var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; + + var vfs: VFS = try .init(std.testing.allocator, testing_io); + defer vfs.deinit(); + const vfs_io = vfs.io(); + + const backend_idx, _, _ = try vfs.lookupDir(.cwd(), "notes:old.txt"); + try std.testing.expectEqual(null, backend_idx); + + const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); + defer dir.close(vfs_io); + const file = try dir.openFile(vfs_io, "notes:old.txt", .{}); + defer file.close(vfs_io); + var contents: [5]u8 = undefined; + try std.testing.expectEqual(contents.len, try file.readPositionalAll(vfs_io, &contents, 0)); + try std.testing.expectEqualStrings("bytes", &contents); + + var real_path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const real_path = real_path_buffer[0..try dir.realPathFile(vfs_io, "notes:old.txt", &real_path_buffer)]; + try std.testing.expect(std.mem.endsWith(u8, real_path, "/notes:old.txt")); +} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 8188c743..bac383e7 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -86,6 +86,16 @@ zig_library( "exe.zig", "floats.zig", "io.zig", + "io/ShardGrid.zig", + "io/Source.zig", + "io/TensorStore.zig", + "io/backend.zig", + "io/buffered_loader.zig", + "io/direct_loader.zig", + "io/dma_calibration.zig", + "io/host_memory.zig", + "io/loader.zig", + "io/log.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/moe/moe.zig", @@ -185,6 +195,11 @@ zig_library( "//platforms/cuda/flashinfer_cutlass_moe", ], "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": [ + "//platforms/rocm", + ], + "//conditions:default": [], }), ) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 3daba2ec..d1455789 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -84,7 +84,8 @@ pub const Options = struct { /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. /// XLA then counts them against device memory, leaves no budget for temporary buffers and /// rematerializes aggressively. Callers set this to the per-device size of such buffers to - /// compensate. Remove once XLA properly supports host-pinned inputs. + /// compensate: `zml.io.TensorStore.View.hostMappedBytes` for those the loader delivers + /// `.host_mapped`. Remove once XLA properly supports host-pinned inputs. extra_device_memory_bytes: u64 = 0, // Debugging options program_name: []const u8 = "zml", @@ -423,7 +424,10 @@ pub fn compile( ) Error!Exe { return switch (platform.io_impl) { .threaded => compileInternal(allocator, io, platform, func_, args, opts), - .zio => try zio.blockInPlace(struct { + // A compile holds its thread for its whole run, so it takes a pool + // worker of its own instead of queuing behind other jobs; concurrent + // compiles run in parallel. + .zio => try zio.blockInPlaceReserved(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const Platform, args_: @TypeOf(args), opts_: Options) Error!Exe { return try compileInternal(allocator_, io_, platform_, func_, args_, opts_); } @@ -448,7 +452,7 @@ pub fn compileInternal( // in mind we might want to revisit that later. // // This is NOT compatible with std.Io.Evented. - // It is compatible with zio, hence the blockInPlace above. + // It is compatible with zio, hence the blockInPlaceReserved above. const span_name = try tracer.formatSpanName(allocator, "zml.module.compile", .{ .program_name = opts.program_name, @@ -802,7 +806,6 @@ fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { // even if later it means that some mesh axis is never used. // Otherwise a "sharded" tensor can in fact be replicated because the underlying mesh has replicas // TODO(#914): decide if we should forbid this altogether - // currently DirectMemoryWriter tests have replicated meshes depending on the platform they target log.warn("Mesh({s}) has numReplicas() == {d}. This imply that all tensor put on this mesh will be implicitly replicated {d} times.\n{f}", .{ m.name, num_replicas, num_replicas, m }); } log.debug("Using {f}", .{m}); @@ -1016,7 +1019,6 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load platform.pjrt_client, platform.pjrt_api, compiler.allocator, - compiler.io, compiler.module, try upb.serialize(options, upb_arena), ); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 2637f209..bc5e6afe 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -405,90 +405,122 @@ pub const Partitioning = packed struct { try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); } - /// Parse a struct literal into a partitioning object. - /// The struct passed to partitioning must only have fields names present in `tags`. - /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. - /// The default value is important. - /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. - pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { - const logical = if (@TypeOf(partitioning) == Logical) - partitioning - else - // capture all the comptime logic in the Logical struct. - comptime Logical.parse(partitioning); - - return logical.applyUseAll(tags, mesh, default_value); - } - + /// A reusable, mesh-independent partitioning keyed by tensor axis tags. + /// Values name logical mesh axes (eg `.model`), or are `.replicated` / `.open`. + /// It is resolved into a positional `Partitioning` once the tensor tags and the mesh are known, + /// so the same value can be applied to tensors with reordered axes, or on different meshes. pub const Logical = struct { - src: Shape.TagsArray, - tgt: Shape.TagsArray, + pub const Spec = union(enum) { + /// Sharded along the logical mesh axis with this name. + axis: Shape.Tag, + replicated, + open, - const replicated: Shape.Tag = Shape.toTag(.replicated); - const open: Shape.Tag = Shape.toTag(.open); - - pub fn parse(v: anytype) Logical { - const V = @TypeOf(v); - - var src_: Shape.TagsArray = .empty; - var tgt_: Shape.TagsArray = .empty; - - if (comptime stdx.meta.isStruct(V)) { - const field_names = std.meta.fieldNames(V); - stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); - inline for (field_names) |field_name| { - const fv = @field(v, field_name); - tgt_.appendAssumeCapacity(@tagName(fv)); - - if (!comptime stdx.meta.isTuple(V)) { - src_.appendAssumeCapacity(field_name); - } - } - return .{ .src = src_, .tgt = tgt_ }; - } - - stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); - } - - test "Logical.parse" { - const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); - - try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.src.constSlice()); - try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.tgt.constSlice()); - } - - pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { - var partition_: Partitioning = .repeat(default_value, tags.len); - var missing_src_axis: ?u8 = null; - - for (0.., logical.src.slice(), logical.tgt.slice()) |i, shape_tag, mesh_tag| { - const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { - missing_src_axis = @intCast(i); - continue; + pub fn init(value: anytype) Spec { + const V = @TypeOf(value); + if (V == Spec) return value; + if (V == @EnumLiteral()) return switch (value) { + .replicated => .replicated, + .open => .open, + else => .{ .axis = Shape.toTag(value) }, }; - - const spec: PartitionSpec = if (mesh_tag == Logical.replicated) - .replicated - else if (mesh_tag == Logical.open) - .open - else spec: { - const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { - stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); - }; - break :spec .sharded(@intCast(mesh_ax)); - }; - partition_ = partition_.set(shape_ax, spec); + if (V == Shape.Tag) return .{ .axis = value }; + stdx.debug.compileError("Partitioning.Logical.Spec expects a logical axis tag, .replicated or .open, got {}", .{V}); } - return .{ partition_, missing_src_axis }; + + pub fn resolve(self: Spec, mesh: *const Mesh) PartitionSpec { + return switch (self) { + .replicated => .replicated, + .open => .open, + .axis => |t| .sharded(@intCast(mesh.resolveLogicalAxis(t) orelse { + stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, t }); + })), + }; + } + }; + + const Entry = struct { tag: Shape.Tag, spec: Spec }; + + entries: stdx.BoundedArray(Entry, MAX_RANK) = .empty, + /// Spec of the axes without an entry. `null` defers to the default of the caller. + default: ?Spec = null, + + pub const replicated: Logical = .{ .default = .replicated }; + + /// Accepts a `Logical`, `.replicated`, or a struct mapping tensor axis tags to specs, + /// eg `.{ .out = .model, .in = .replicated }`. Null optional specs are skipped. + pub fn init(specs: anytype) Logical { + const T = @TypeOf(specs); + if (T == Logical) return specs; + if (T == @EnumLiteral()) return switch (specs) { + .replicated => Logical.replicated, + else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), + }; + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Partitioning.Logical expects a struct of partition specs or .replicated, got {}", .{T}); + + var result: Logical = .{}; + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const value = @field(specs, field_name); + const spec: ?Spec = if (@typeInfo(@TypeOf(value)) == .optional) (if (value) |v| Spec.init(v) else null) else Spec.init(value); + if (spec) |s| result.entries.append(.{ .tag = Shape.toTag(field_name), .spec = s }) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + } + return result; } - pub fn applyUseAll(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { - const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); - - if (missing_shape_axis) |ax| { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.src.get(ax) }); + /// Returns the spec of the given tensor axis, or `default` when it has no entry. + pub fn get(self: Logical, tag_: anytype) ?Spec { + const axis_tag = Shape.toTag(tag_); + for (self.entries.constSlice()) |entry| { + if (std.mem.eql(u8, std.mem.span(entry.tag), std.mem.span(axis_tag))) return entry.spec; } - return p; + return self.default; + } + + /// Resolves the partitioning of a tensor with the given axis tags on `mesh`. + /// Every entry must name one of `tags`. + pub fn resolve(self: Logical, tags: []const Shape.Tag, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { + var res: Partitioning = .repeat(if (self.default) |d| d.resolve(mesh) else default_value, tags.len); + for (self.entries.constSlice()) |entry| { + const ax = Shape.axisFromTagMaybe(tags, entry.tag) orelse { + stdx.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), entry.tag }); + }; + res = res.set(ax, entry.spec.resolve(mesh)); + } + return res; + } + + test "Logical resolves by tensor axis name on any mesh" { + const tp: Mesh = .initTest(.tp, .{.model}); + const ep: Mesh = .initTest(.ep, .{ .experts, .model }); + const specs: []const Spec = &.{ .init(.model), .replicated, .open }; + for (specs) |spec| { + const logical: Logical = .init(.{ .out = spec, .in = .replicated }); + const out = logical.get(.out).?; + try std.testing.expectEqual(std.meta.activeTag(spec), std.meta.activeTag(out)); + try std.testing.expectEqual(spec.resolve(&ep), out.resolve(&ep)); + try std.testing.expectEqual(null, logical.get(.batch)); + + const tags: []const Shape.Tag = &.{ "in", "batch", "out" }; + try std.testing.expectEqualSlices( + PartitionSpec, + &.{ .replicated, .open, spec.resolve(&tp) }, + logical.resolve(tags, &tp, .open).toArray()[0..3], + ); + try std.testing.expectEqual(spec.resolve(&ep), logical.resolve(tags, &ep, .replicated).get(2)); + } + try std.testing.expectEqual(PartitionSpec.sharded(1), Spec.init(.model).resolve(&ep)); + } + + test "Logical replicated default and optional specs" { + const tp: Mesh = .initTest(.tp, .{.model}); + const logical: Logical = .init(.replicated); + try std.testing.expect(logical.get(.out).? == .replicated); + try std.testing.expectEqual(Partitioning.replicated(2), logical.resolve(&.{ "a", "b" }, &tp, .open)); + + const maybe: ?Spec = null; + const partial: Logical = .init(.{ .a = maybe, .b = @as(?Spec, .init(.model)) }); + try std.testing.expectEqual(1, partial.entries.len); + try std.testing.expectEqual(Partitioning.init(&.{ .open, .mesh_axis_0 }), partial.resolve(&.{ "a", "b" }, &tp, .open)); } }; }; diff --git a/zml/buffer.zig b/zml/buffer.zig index 8714d80f..64d520a7 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -27,8 +27,8 @@ test { /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. /// /// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. -/// * loading weights from disk directly to the `device zml.aio.loadBuffers` -/// * can be created by calling `HostBuffer.toDevice(platform)`. +/// * filled from a checkpoint by `zml.io.Loader` (docs/learn/loader.md). +/// * created from host data by `fromSlice` or `fromBytes`. pub const Buffer = struct { _platform: *const Platform, _shape: Shape, @@ -56,6 +56,16 @@ pub const Buffer = struct { pub fn devicePtr(self: *const Shard) *anyopaque { return self._pjrt_buffer.opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; } + + /// Copies this shard's bytes to `destination`, which must hold the + /// shard's byte size, and waits for the copy. + pub fn toHost(self: *const Shard, io: std.Io, destination: []u8) !void { + const maybe_event = try self._pjrt_buffer.toHostBuffer(self._platform.pjrt_api, destination); + if (maybe_event) |event| { + defer event.deinit(self._platform.pjrt_api); + try event.await(self._platform.pjrt_api, io); + } + } }; pub const ShardIterator = struct { @@ -99,17 +109,6 @@ pub const Buffer = struct { }.deinit, {}, buffers); } - /// Given an arbitrary struct `deinit` all `zml.Buffer` containing. - /// If the struct contains slices of `zml.Buffer` the memory of the slices will NOT be freed, - /// This only impacts device memory. - pub fn freeDeviceMemoryButKeepHostMetadataMemory(T: type, buffers: *mem.Bufferized(T)) void { - meta.visit(struct { - fn deinit(_: void, x: *Buffer) void { - x.deinit(); - } - }.deinit, {}, buffers); - } - /// This Buffer shape. pub fn shape(self: Buffer) Shape { return self._shape; @@ -119,6 +118,8 @@ pub const Buffer = struct { return @intCast(self._shards.len); } + /// One shard per device of the shape's mesh (the platform's replicated + /// mesh when it names none), in that mesh's canonical device order. pub fn shards(self: *const Buffer) ShardIterator { return .{ ._platform = self._platform, @@ -156,7 +157,6 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); const byte_size: usize = if (packed_shape.rank() == 0) packed_shape.dtype().sizeOf() else @@ -164,8 +164,8 @@ pub const Buffer = struct { stdx.debug.assert(data_.len == byte_size, "Expected data length {}, got {}", .{ byte_size, data_.len }); const base_ptr = data_.ptr; - for (platform.physical_mesh.devices_in_canonical_order, 0..) |device, device_index| { - const memory = platform.devices[device.id].memory(opts.memory).?; + for (placement.mesh().?.devicesInCanonicalOrder(), 0..) |device, device_index| { + const memory = deviceMemory(platform, device, opts.memory); const args: pjrt.Client.BufferFromHostBufferArgs = .{ // Change for each device .data = placement.shardPtr(device_index, base_ptr), @@ -248,10 +248,8 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); - - for (platform.physical_mesh.devices_in_canonical_order) |device| { - const memory = platform.devices[device.id].memory(opts.memory).?; + for (metadata.placement.mesh().?.devicesInCanonicalOrder()) |device| { + const memory = deviceMemory(platform, device, opts.memory); const args: pjrt.Client.CreateUninitializedBufferArgs = .{ // Change for each device .dst = .{ .memory = memory.pjrt_memory }, @@ -295,14 +293,14 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); const placement = placementOrPanic(self._platform, self._shape); - for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + for (self._shards.constSlice(), 0..) |shard, shard_index| { // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed const sub_slice = placement.shardSlice(shard_index, slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; const size_bytes = placement.localShape(self._shape).byteSize(); const destination = sub_slice.data()[0..size_bytes]; - const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); + const maybe_event = try shard.toHostBuffer(self._platform.pjrt_api, destination); if (maybe_event) |event| { defer event.deinit(self._platform.pjrt_api); @@ -322,9 +320,9 @@ pub const Buffer = struct { var shard_slice = try Slice.alloc(allocator, placement.localShape(self._shape)); defer shard_slice.free(allocator); - for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + for (self._shards.constSlice(), 0..) |shard, shard_index| { const sub_slice = placement.shardSlice(shard_index, slice); - const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); + const maybe_event = try shard.toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { defer event.deinit(self._platform.pjrt_api); try event.await(self._platform.pjrt_api, io); @@ -421,6 +419,41 @@ test "device round-trip" { } } +test "a buffer on a sub-mesh has one shard per device of that mesh" { + const io = std.testing.io; + const allocator = std.testing.allocator; + // Four devices: the accelerator's when it has them, else a CPU platform's + // (none on a host without the CPU plugin, such as aarch64 Linux). + var devices: testing.Devices = undefined; + devices.init(allocator, io, 4) catch return error.SkipZigTest; + defer devices.deinit(); + const platform = devices.platform; + // Two of the four devices, out of the platform's order. + const nodes = [_]Sharding.PhysicalNode{ .device(platform.devices[3]), .device(platform.devices[1]) }; + var physical: Sharding.PhysicalMesh = try .fromTree(allocator, platform.target, .axis(.bus, .tree, &nodes)); + defer physical.deinit(allocator); + const mesh: Sharding.Mesh = try .init("pair", &physical, .mesh(.{ .model = .high_bandwidth }), .parseBindings(.{ .model = .bus })); + + var values: [8]u32 = undefined; + for (&values, 0..) |*value, i| value.* = @intCast(i); + const shape = Shape.init(.{ .b = 4, .d = 2 }, .u32).withPartitioning(&mesh, .{ .b = .model }); + var buffer: Buffer = try .fromBytes(io, platform, shape, std.mem.sliceAsBytes(&values)); + defer buffer.deinit(); + try testing.expectShards(allocator, io, &buffer, std.mem.sliceAsBytes(&values), .default); + + const read: Slice = try buffer.toSliceAlloc(allocator, io); + defer read.free(allocator); + try std.testing.expectEqualSlices(u32, &values, read.constItems(u32)); + var into: [8]u32 = undefined; + try buffer.toSlice(io, .init(shape, std.mem.sliceAsBytes(&into))); + try std.testing.expectEqualSlices(u32, &values, &into); + + const pinned: Buffer.HostAccessible = try .init(allocator, io, platform, shape, .undonatable); + defer pinned.deinit(allocator); + pinned.view(u32).copyFrom(&values); + try testing.expectShards(allocator, io, &pinned.buffer, std.mem.sliceAsBytes(&values), .host_pinned); +} + pub const PjrtBufferMetadata = struct { shape: Shape, ty: pjrt.BufferType, @@ -446,10 +479,16 @@ pub const PjrtBufferMetadata = struct { } }; +/// The `kind` memory of `device`, a device of a buffer's mesh, which names it +/// by its index in `platform.devices`. Outside `Buffer` for `Buffer.HostAccessible`. +pub fn deviceMemory(platform: *const Platform, device: Sharding.Device, kind: Memory.Kind) *const Memory { + stdx.debug.assert(device.id < platform.devices.len, "Mesh device {d} is not one of the {d} devices of {t}", .{ device.id, platform.devices.len, platform.target }); + return platform.devices[device.id].memory(kind) orelse stdx.debug.panic("Device {d} doesn't have {t} memory", .{ device.id, kind }); +} + fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { - var sh = shape; - if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; - return Sharding.Placement.init(&sh) catch |err| { + const placed = platform.placedShape(shape); + return Sharding.Placement.init(&placed) catch |err| { @branchHint(.cold); std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); }; diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig index b3e82a9c..20d90657 100644 --- a/zml/buffer/HostAccessible.zig +++ b/zml/buffer/HostAccessible.zig @@ -17,6 +17,7 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("../buffer.zig").Buffer; +const deviceMemory = @import("../buffer.zig").deviceMemory; const PjrtBufferMetadata = @import("../buffer.zig").PjrtBufferMetadata; const Platform = @import("../platform.zig").Platform; const Shape = @import("../shape.zig").Shape; @@ -249,7 +250,7 @@ pub fn init( ._shards = .empty, }; const metadata: PjrtBufferMetadata = .init(platform, shape_); - const devices = platform.physical_mesh.devices_in_canonical_order; + const devices = metadata.placement.mesh().?.devicesInCanonicalOrder(); const api = platform.pjrt_api; var allocations: HostPinnedAllocations = .empty; @@ -259,7 +260,7 @@ pub fn init( errdefer allocations.deinitHandles(api, buffer._shards.constSlice()); for (devices, 0..) |device, shard_index| { - const memory = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; + const memory = deviceMemory(platform, device, .host_pinned).pjrt_memory; const origin: Coordinates = .ofShard(&metadata.placement, shard_index, metadata.shape); const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.localShape(metadata.shape)) else null; diff --git a/zml/exe.zig b/zml/exe.zig index 85baa7e7..179df29f 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -310,7 +310,7 @@ pub const Exe = struct { /// Non-donated inputs remain owned by the caller. pub fn run(self: *Runner, io: std.Io, input_values: anytype, output_refs: anytype, opts: CallOpts) void { self.args.set(input_values); - self.exe.call(io, self.args, &self.results, opts); + self.exe.call(io, self.args, &self.results, opts) catch |err| std.debug.panic("PJRT execution failed with: {}", .{err}); self.exe.deinitDonatedInputs(self.args); self.results.fill(output_refs); } @@ -334,7 +334,9 @@ pub const Exe = struct { /// Executes with optional waiting. The caller owns all input handles, including donated inputs. /// Donated input contents cannot be used after the call; output buffers have their own handles. /// Use a runner to destroy donated handles automatically and replace buffers through output destinations. - pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + /// A failed launch writes no outputs; with `wait`, a failed execution destroys the outputs it + /// wrote, so `results_` holds no buffer after any error. + pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) pjrt.ApiError!void { var span = tracer.span("zml.exe.call", .{ .wait = opts.wait, .arg_count = arguments.expected_shapes.len, @@ -342,6 +344,7 @@ pub const Exe = struct { }); defer span.end(); + const api = self.platform.pjrt_api; var events: [Platform.MAX_NUM_DEVICES]?*pjrt.Event = @splat(null); const partition_events = events[0..@intCast(self.num_partitions)]; @@ -350,7 +353,7 @@ pub const Exe = struct { .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) partition_events else null, }; - self.exe.execute(self.platform.pjrt_api, .{ + try self.exe.execute(api, .{ .arguments = arguments.flat_buffers.buffers, .num_args = arguments.expected_shapes.len, .results = results_.flat_buffers.buffers, @@ -358,33 +361,20 @@ pub const Exe = struct { // TODO: Set this flag for undonatable buffers. .non_donatable_input_indices = &.{}, .context = self.context, - }) catch |err| { - std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); - }; + }); - switch (self.platform.target) { - .neuron => { - for (events_slice.?) |e| { - if (e) |ev| { - if (opts.wait) { - ev.await(self.platform.pjrt_api, io) catch |err| { - std.debug.panic("PJRT execution failed with: {}", .{err}); - }; - } - ev.deinit(self.platform.pjrt_api); - } - } - }, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) { - for (events_slice.?) |e| { - if (e) |ev| { - ev.await(self.platform.pjrt_api, io) catch |err| { - std.debug.panic("PJRT execution failed with: {}", .{err}); - }; - ev.deinit(self.platform.pjrt_api); - } - } - }, + // Every partition's event is destroyed, awaited first when waiting. + var failure: ?pjrt.ApiError = null; + if (events_slice) |slice| for (slice) |e| { + const ev = e orelse continue; + if (opts.wait) ev.await(api, io) catch |err| { + failure = failure orelse err; + }; + ev.deinit(api); + }; + if (failure) |err| { + for (results_.flat_buffers.raw_buffers) |buffer| buffer.deinit(api); + return err; } } }; @@ -470,7 +460,7 @@ pub fn FnExe(comptime function_: anytype) type { /// Destroys donated input handles before filling outputs; non-donated inputs remain caller-owned. pub fn run(self: *RunnerSelf, io: std.Io, call: Call, opts: Exe.CallOpts) void { self.args.set(call.inputs); - self.exe.call(io, self.args, &self.results, opts); + self.exe.call(io, self.args, &self.results, opts) catch |err| std.debug.panic("PJRT execution failed with: {}", .{err}); self.exe.deinitDonatedInputs(self.args); self.results.fill(call.outputs); } @@ -620,7 +610,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); args.set(.{ value_buffer, weight_buffer }); - exe.call(io, args, &results, .{ .wait = wait }); + try exe.call(io, args, &results, .{ .wait = wait }); var output = results.get(Buffer); defer output.deinit(); @@ -628,6 +618,8 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { for (value_buffer._shards.constSlice()) |shard| { try std.testing.expect(shard.isDeleted(platform.pjrt_api)); } + // Executing over the deleted input is an error, not a panic. + try std.testing.expect(std.meta.isError(exe.call(io, args, &results, .{ .wait = true }))); value_buffer.deinit(); try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); try std.testing.expectEqual([2]i32{ 10, 20 }, try weight_buffer.getValue([2]i32, io)); diff --git a/zml/io.zig b/zml/io.zig index 43bfdf23..837b1f1e 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1,1706 +1,32 @@ +//! Checkpoint loading and virtual filesystem integration. The user guide is +//! docs/learn/loader.md; each file's header holds its internals: +//! +//! - io/loader.zig: options and their checks, the set-up task, starts, +//! program compiles and groups, admission, and the one check of every +//! binding before the backends see it (`LoadSpec`). +//! - io/backend.zig: backend selection, the request size rule and the +//! `LoadSpec` contract. +//! - io/TensorStore.zig, io/Source.zig: bindings, views, and the sources +//! that describe a tensor's checkpoint bytes. +//! - io/direct_loader.zig: sizing, per-file planning, FIFO claims, source +//! workers, per-device pumps and transfer completion. +//! - io/ShardGrid.zig: where a destination's logical bytes land. +//! - io/buffered_loader.zig: whole-tensor staging and positional reads. +//! - io/dma_calibration.zig: DMA block selection. +//! - io/host_memory.zig: pinned host arenas and their blocks. +//! - io/log.zig: log scopes. Loader test support is in `zml.testing`. +//! - vfs: local and remote files behind one `std.Io`, the remote error +//! contract, retries and the rate-limit hold (vfs/request.zig). + const std = @import("std"); -const pjrt = @import("pjrt"); -const stdx = @import("stdx"); pub const VFS = @import("vfs"); -const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("zml.zig").Bufferized; -const CreateOptions = @import("platform.zig").CreateOptions; -const Exe = @import("exe.zig").Exe; -const mem = @import("mem.zig"); -const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); -const pjrtx = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const tracer = @import("profiling/tracer.zig"); -const safetensors = @import("safetensors.zig"); -const Shape = @import("shape.zig").Shape; -const Sharding = @import("Sharding.zig"); -const Placement = Sharding.Placement; -const Slice = @import("slice.zig").Slice; -const Tensor = @import("tensor.zig").Tensor; +pub const Loader = @import("io/loader.zig").Loader; +pub const TensorStore = @import("io/TensorStore.zig"); -const log = std.log.scoped(.@"zml/io"); +pub const dma_calibration = @import("io/dma_calibration.zig"); -pub const TensorStore = struct { - pub const Binding = struct { - tensors: []*safetensors.Tensor, - transformed: bool, - memory: Memory.Kind = .default, - }; - - registry: *safetensors.TensorRegistry, - meshes: []const *const Sharding.Mesh, - id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - - pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { - var arena: std.heap.ArenaAllocator = .init(allocator); - return .{ - .registry = registry, - .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), - .id_to_sources = .empty, - .allocator = allocator, - .arena = arena, - }; - } - - pub fn deinit(self: *TensorStore) void { - self.id_to_sources.deinit(self.allocator); - self.arena.deinit(); - } - - fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { - const gop = try self.id_to_sources.getOrPut(self.allocator, id); - if (gop.found_existing) { - stdx.debug.panic("Id {} already has associated sources", .{id}); - } - errdefer self.id_to_sources.removeByPtr(gop.key_ptr); - - gop.value_ptr.* = sources; - } - - fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*safetensors.Tensor { - const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; - return tensor_desc_ptr; - } - - fn dupeSource(self: *TensorStore, key: []const u8) ?*safetensors.Tensor { - const entry = self.getPtrFromKey(key) orelse return null; - - const copy = self.arena.allocator().create(safetensors.Tensor) catch @panic("OOM"); - copy.* = entry.*; - - return copy; - } - - fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { - const sources = self.id_to_sources.get(id) orelse return null; - stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); - return sources.tensors[0]; - } - - pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { - return self.registry.reader(io, key, buffer); - } - - pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { - const sources = self.id_to_sources.get(id) orelse return error.NotFound; - stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); - - return sources.tensors[0].reader(io, buffer, .{}); - } - - pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?Binding { - return self.id_to_sources.get(id); - } - - pub fn getShape(self: *const TensorStore, key: []const u8) ?Shape { - const entry_ptr = self.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; - } - - pub fn view(self: *TensorStore) View { - return .{ .store = self }; - } - - pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { - const name_slice = @tagName(name); - for (store.meshes) |m| { - if (std.mem.eql(u8, name_slice, m.name)) { - return m; - } - } - std.debug.panic( - \\Found no meshes named {s} in TensorStore. - \\Try passing more meshes to `zml.TensorStore.fromRegistry`. - \\Known meshes: {f} - , .{ name_slice, stdx.fmt.slice(store.meshes) }); - } - - pub const View = struct { - store: *TensorStore, - - prefix_buffer: [256]u8 = undefined, - prefix_length: usize = 0, - - pub fn root(self: *const View) View { - return .{ - .store = self.store, - }; - } - - pub fn parent(self: *const View) View { - const slice = self.prefix() orelse unreachable; - const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); - var buffer: [256]u8 = undefined; - @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = index + 1, - }; - } - - pub fn withPrefix(self: *const View, prefix_: []const u8) View { - var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }); - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - pub fn withLayer(self: *const View, index: usize) View { - var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }); - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - pub fn prefix(self: *const View) ?[]const u8 { - return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; - } - - pub fn hasKey(self: *const View, subkey: []const u8) bool { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - return for (self.store.registry.tensors.keys()) |k| { - if (std.mem.startsWith(u8, k, key)) break true; - } else false; - } - - /// Creates a zml.Tensor from a specific entry in the store. - pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { - return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { - const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); - const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; - if (mesh_ == .replicated) { - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .replicated); - } - const resolved_mesh = self.store.mesh(mesh_); - // Unspecified tensor axes are treated as replicated - const partition: Sharding.Partitioning = if (has_tags) - .parse(parsed_tags.constSlice(), resolved_mesh, .replicated, partitioning) - else - partitioning; - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .{ .mesh = resolved_mesh, .partition = partition }); - } - - pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { - return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { - const tensor = self.maybeCreateTensor(subkey, tags, mesh_, partitioning); - if (tensor) |t| { - const storage = self.store.id_to_sources.getPtr(t.id); - storage.?.memory = .host_pinned; - } - return tensor; - } - - pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { - const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); - const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; - - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.slice() else null, .replicated) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - const source = self.store.dupeSource(key) orelse return null; - - const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); - errdefer self.store.arena.allocator().free(sources); - sources[0] = source; - - var shape = source.shape; - if (tags) |user_tags| { - stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); - @memcpy(shape._tags.slice(), user_tags); - } - shape._sharding = sharding; - if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); - - const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); - - return tensor; - } - - pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { - const arena = self.store.arena.allocator(); - - var tensor_list = std.ArrayList(*safetensors.Tensor).initCapacity(arena, sources.len) catch |e| std.debug.panic("Not handling {} errors", .{e}); - defer tensor_list.deinit(arena); - - var buffer: [256]u8 = undefined; - for (sources) |subkey| { - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - const tensor = self.store.dupeSource(key) orelse return null; - tensor_list.appendAssumeCapacity(tensor); - } - - const tensors = tensor_list.toOwnedSlice(arena) catch unreachable; - errdefer arena.free(tensors); - - const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = tensors, .transformed = true }) catch |e| std.debug.panic("Not handling {} errors", .{e}); - - return tensor; - } - - pub fn getShape(self: View, subkey: []const u8) ?Shape { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - return self.store.getShape(key); - } - - pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { - var buffer: [256]u8 = undefined; - const key = if (opts.no_prefix) - subkey - else b: { - break :b makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - }; - return self.store.getShape(key); - } - - pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { - var key_buffer: [256]u8 = undefined; - const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - return self.store.getReader(key, io, buffer); - } - - pub fn count(self: View) usize { - var count_: usize = 0; - const prefix_ = self.prefix() orelse ""; - var it = self.store.registry.tensors.iterator(); - while (it.next()) |item| { - const key = item.key_ptr.*; - if (std.mem.startsWith(u8, key, prefix_)) { - count_ += 1; - } - } - return count_; - } - - fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { - const key = std.fmt.bufPrint(buffer, fmt, args) catch - std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); - return key; - } - }; -}; - -pub const Loader = struct { - allocator: std.mem.Allocator, - platform: *const Platform, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - pinned_buffer_pools: []mem.DynamicBufferPool, - group: stdx.Io.LimitedGroup, - bytes_loaded: std.atomic.Value(usize) = .init(0), - delivered: std.AutoHashMapUnmanaged(Tensor.Id, void) = .empty, - - pub const Opts = struct { - pub const default: Opts = .{ - .parallelism = 1, - .dma_chunks = 2, - .dma_chunk_size = 4096, - }; - parallelism: usize, - dma_chunks: usize, - dma_chunk_size: usize, - }; - - pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!Loader { - const pool_count = platform.devices.len; - const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); - errdefer allocator.free(dma_allocators); - for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(allocator, device); - } - - const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); - errdefer allocator.free(buffer_pools); - for (buffer_pools) |*pool_| { - pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); - } - errdefer for (buffer_pools, 0..) |*pool_, i| { - pool_.deinit(dma_allocators[i].allocator()); - }; - - return .{ - .allocator = allocator, - .platform = platform, - .dma_allocators = dma_allocators, - .dma_chunk_size = opts.dma_chunk_size, - .pinned_buffer_pools = buffer_pools, - .group = .init(opts.parallelism), - }; - } - - pub fn deinit(self: *Loader) void { - self.delivered.deinit(self.allocator); - for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); - self.allocator.free(self.pinned_buffer_pools); - self.allocator.free(self.dma_allocators); - } - - pub fn await(self: *Loader, io: std.Io) std.Io.Cancelable!void { - return self.group.await(io); - } - - pub const LoadOpts = struct { - progress: ?*std.Progress.Node = null, - }; - - pub const LoadError = error{TransformedTensorNotDelivered}; - - pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { - const tensor_count = meta.count(Tensor, model); - - var arena: std.heap.ArenaAllocator = .init(self.allocator); - defer arena.deinit(); - - const flattened_buffers = arena.allocator().alloc(*Buffer, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); - meta.forEachVisit(buffers, *Buffer, struct { - fn call(i: usize, buffer: *Buffer, flattened_buffers_: []*Buffer) void { - flattened_buffers_[i] = buffer; - } - }.call, .{flattened_buffers}); - - const flattened_tensors = arena.allocator().alloc(*const Tensor, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); - meta.forEachVisit(model, *const Tensor, struct { - fn call(i: usize, tensor: *const Tensor, flattened_tensors_: []*const Tensor) void { - flattened_tensors_[i] = tensor; - } - }.call, .{flattened_tensors}); - - // Refuse before scheduling anything, so a failure leaves no task in flight. - var missing: usize = 0; - for (flattened_tensors) |tensor| { - const sources = store.getSourcesById(tensor.id) orelse continue; - if (!sources.transformed or self.delivered.contains(tensor.id)) { - continue; - } - - missing += 1; - logNotDelivered(arena.allocator(), tensor, sources.tensors); - } - - if (missing != 0) { - return error.TransformedTensorNotDelivered; - } - - for (flattened_tensors, flattened_buffers) |tensor, buffer| { - if (store.getSourcesById(tensor.id)) |sources| { - if (sources.transformed) continue; - } - - self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); - } - } - - fn logNotDelivered(arena: std.mem.Allocator, tensor: *const Tensor, sources: []const *safetensors.Tensor) void { - const max_names = 8; - var names: std.Io.Writer.Allocating = .init(arena); - for (sources[0..@min(sources.len, max_names)], 0..) |source, i| { - names.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break; - } - - if (sources.len > max_names) { - names.writer.print(" and {} more", .{sources.len - max_names}) catch {}; - } - - log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); - } - - pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { - loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); - } - - fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { - const sources = store.getSourcesById(tensor.id) orelse { - std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); - return; - }; - stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); - - self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { - log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); - unreachable; - }; - } - - fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { - self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { - log.err("Failed to load tensor {s}: {}", .{ source.name, e }); - loaded.* = false; - return; - }; - loaded.* = true; - } - - fn loadSingleInner( - self: *Loader, - io: std.Io, - source: *safetensors.Tensor, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - opts: LoadOpts, - ) !void { - var reader = try source.reader(io, &.{}, .{}); - defer reader.deinit(); - - var writer = try MemoryWriter.init( - self.allocator, - io, - self.platform, - self.pinned_buffer_pools, - self.dma_allocators, - self.dma_chunk_size, - shape, - buffer, - memory, - ); - defer writer.deinit(self.allocator); - - const scale = 1024; - - if (opts.progress) |progress| { - var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); - defer node.end(); - writer.setProgress(&node); - defer writer.setProgress(null); - var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); - const total = try reader.interface.streamRemaining(&progress_writer.interface); - try progress_writer.interface.flush(); - _ = self.bytes_loaded.fetchAdd(total, .monotonic); - } else { - const total = try reader.interface.streamRemaining(writer.interface()); - try writer.interface().flush(); - _ = self.bytes_loaded.fetchAdd(total, .monotonic); - } - } - - pub fn loadExecute( - self: *Loader, - arena: std.mem.Allocator, - io: std.Io, - tensor: Tensor, - buffer: *Buffer, - store: *const TensorStore, - exe: *const Exe, - opts: LoadOpts, - ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { - const sources = (store.getSourcesById(tensor.id) orelse return error.NotFound).tensors; - const buffers = try arena.alloc(Buffer, sources.len); - const loaded = try arena.alloc(bool, sources.len); - @memset(loaded, false); - defer for (buffers, loaded) |*b, l| if (l) b.deinit(); - - var node = if (opts.progress) |progress| p: { - var writer = std.Io.Writer.Allocating.init(arena); - print: { - writer.writer.writeAll("Running executable on ") catch break :print; - for (sources, 0..) |source, i| { - writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break :print; - } - } - break :p progress.start(writer.written(), 1); - } else null; - defer if (node) |*n| n.end(); - - for (sources, 0..) |source, i| { - self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); - } - self.group.await(io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - }; - - if (std.mem.findScalar(bool, loaded, false)) |_| { - return error.LoadFailed; - } - - var args = try exe.args(arena); - var results = try exe.results(arena); - - args.set(.{buffers}); - exe.call(io, args, &results, .{ .wait = true }); - - buffer.* = results.get(Buffer); - try self.delivered.put(self.allocator, tensor.id, {}); - } -}; - -test "Loader.loadExecute cleans up donated and non-donated inputs" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = @import("testing.zig").env(); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); - const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); - defer allocator.free(path); - - const shape: Shape = .init(.{2}, .i32); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); - try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); - defer store.deinit(); - var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); - defer loader.deinit(); - - inline for (.{ false, true }) |donate| { - const Functions = struct { - fn forward(value: Tensor, weight: Tensor) Tensor { - const sum = value.add(weight); - return if (donate) sum.reuseBuffer(value) else sum; - } - }; - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const tensor = store.view().maybeCreateBinding(&.{ "value", "weight" }, shape) orelse unreachable; - const exe = try platform.compileFn(allocator, io, Functions.forward, .{ Tensor.fromShape(shape), Tensor.fromShape(shape) }, .{}); - defer exe.deinit(); - try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); - - var output: Buffer = undefined; - try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); - defer output.deinit(); - try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); - try std.testing.expect(loader.delivered.contains(tensor.id)); - } -} - -pub const ProgressWriter = struct { - inner: *std.Io.Writer, - progress: *std.Progress.Node, - interface: std.Io.Writer, - total: usize = 0, - scale: usize, - - pub const InitOpts = struct { - scale: usize = 1, - }; - - pub fn init(inner_: *std.Io.Writer, progress_: *std.Progress.Node, opts: InitOpts) ProgressWriter { - return .{ - .inner = inner_, - .progress = progress_, - .scale = opts.scale, - .interface = .{ - .buffer = inner_.buffer, - .end = inner_.end, - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - .sendFile = sendFile, - }, - }, - }; - } - - pub fn pre(self: *ProgressWriter) usize { - self.inner.buffer = self.interface.buffer; - self.inner.end = self.interface.end; - return self.inner.end; - } - - pub fn post(self: *ProgressWriter, len_pre: usize, total: usize) void { - self.interface.buffer = self.inner.buffer; - self.interface.end = self.inner.end; - const drained_pre = len_pre -| self.interface.end; - self.total += drained_pre + total; - self.progress.setCompletedItems(self.total / self.scale); - } - - pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - errdefer self.post(len_pre, 0); - const total = try self.inner.vtable.drain(self.inner, data, splat); - self.post(len_pre, total); - return total; - } - - pub fn sendFile(w: *std.Io.Writer, file_reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.Writer.FileError!usize { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - errdefer self.post(len_pre, 0); - const total = try self.inner.vtable.sendFile(self.inner, file_reader, limit); - self.post(len_pre, total); - return total; - } - - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - defer self.post(len_pre, 0); - try self.inner.vtable.flush(self.inner); - } - - pub fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - defer self.post(len_pre, 0); - try self.inner.vtable.rebase(self.inner, preserve, capacity); - } -}; - -pub const MemoryWriter = union(enum) { - direct: DirectMemoryWriter, - buffered: BufferedMemoryWriter, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - pools: []mem.DynamicBufferPool, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - ) !MemoryWriter { - return switch (platform.target) { - .cuda, .rocm, .oneapi, .furiosa => .{ - .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), - }, - .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) - std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) - else - .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, - }; - } - - pub fn interface(self: *MemoryWriter) *std.Io.Writer { - return switch (self.*) { - .direct => &self.direct.interface, - .buffered => &self.buffered.interface, - }; - } - - pub fn deinit(self: *MemoryWriter, allocator: std.mem.Allocator) void { - switch (self.*) { - .direct => self.direct.deinit(), - .buffered => self.buffered.deinit(allocator), - } - } - - pub fn setProgress(self: *MemoryWriter, progress: ?*std.Progress.Node) void { - switch (self.*) { - .direct => self.direct.setProgress(progress), - .buffered => {}, - } - } -}; - -pub const BufferedMemoryWriter = struct { - io: std.Io, - platform: *const Platform, - shape: Shape, - buffer: *Buffer, - interface: std.Io.Writer, - - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { - return .{ - .io = io, - .platform = platform, - .shape = shape, - .buffer = buffer, - .interface = .{ - .buffer = try allocator.alloc(u8, shape.byteSize()), - .vtable = &.{ - .drain = std.Io.Writer.fixedDrain, - .flush = flush, - .rebase = std.Io.Writer.failingRebase, - }, - }, - }; - } - - pub fn deinit(self: *BufferedMemoryWriter, allocator: std.mem.Allocator) void { - if (self.interface.buffer.len > 0) { - allocator.free(self.interface.buffer); - } - } - - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - self.buffer.* = Buffer.from( - self.io, - self.platform, - self.shape, - @ptrCast(self.interface.buffer), - .{ .wait = true }, - ) catch return std.Io.Writer.Error.WriteFailed; - } -}; - -const DirectShardWriter = struct { - const EventContext = struct { - self: *DirectShardWriter, - err: ?*pjrt.Error = null, - pjrt_event: *pjrt.Event, - event: std.Io.Event = .unset, - buffer: []u8, - }; - - allocator: std.mem.Allocator, - io: std.Io, - memory: *const Memory, - pool: *mem.DynamicBufferPool, - total: usize, - pjrt_buffer: *pjrt.Buffer, - - transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, - offset: usize = 0, - - interface: std.Io.Writer, - flip_flop: u1 = 0, - events_contexts: [2]?EventContext = @splat(null), - - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape) !DirectShardWriter { - const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); - const transfer_manager = try memory.platform.pjrt_client.createBuffersForAsyncHostToDevice( - memory.platform.pjrt_api, - .{ - .shape_specs = &.{shape_spec}, - .memory = memory.pjrt_memory, - }, - ); - errdefer transfer_manager.deinit(memory.platform.pjrt_api); - - const pjrt_buffer = try transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0); - errdefer pjrt_buffer.deinit(memory.platform.pjrt_api); - - const buf = try pool.get(allocator, io); - - return .{ - .allocator = allocator, - .io = io, - .memory = memory, - .pool = pool, - .total = shape.byteSize(), - .pjrt_buffer = pjrt_buffer, - .transfer_manager = transfer_manager, - .interface = .{ - .buffer = buf[0..@min(buf.len, shape.byteSize())], - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - }, - }, - }; - } - - pub fn deinit(self: *DirectShardWriter) void { - self.transfer_manager.deinit(self.memory.platform.pjrt_api); - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - - const chunk = data[0]; - if (chunk.len > self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - if (chunk.len > self.total - (self.offset + self.interface.end)) return std.Io.Writer.Error.WriteFailed; - - const needs_fresh_buffer = chunk.len > self.interface.buffer.len - self.interface.end; - if (needs_fresh_buffer) { - try self.interface.flush(); - } - - @memcpy(self.interface.buffer[self.interface.end..][0..chunk.len], chunk); - self.interface.end += chunk.len; - - const buffer_full = self.interface.end == self.interface.buffer.len; - if (buffer_full) { - try self.interface.flush(); - } - - return chunk.len; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - try self.flushBuffered(); - } - - fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - if (self.interface.buffer.len - self.interface.end >= capacity) return; - if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - try self.interface.flush(); - } - - fn flushBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { - if (self.offset >= self.total) return; - - const pjrt_api = self.memory.platform.pjrt_api; - - const current_buffer = self.interface.buffer; - const buffered = self.interface.buffered(); - - const slice = buffered[0..@min(buffered.len, self.total - self.offset)]; - if (slice.len == 0) return; - const is_last = (self.offset + slice.len) >= self.total; - - const transfer_event = self.transfer_manager.transferData(pjrt_api, 0, slice, @intCast(self.offset), is_last) catch |err| { - log.err("error when transferring data to device: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - - const ctx = &self.events_contexts[@intCast(self.flip_flop)]; - ctx.* = .{ - .self = self, - .buffer = current_buffer, - .pjrt_event = transfer_event, - }; - - transfer_event.onReady(pjrt_api, EventContext, struct { - fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { - ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); - ctx_.err = err; - ctx_.event.set(ctx_.self.io); - } - }.call, &(ctx.*.?)) catch |err| { - log.err("error when setting up transfer completion callback: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - - if (self.events_contexts[@intCast(self.flip_flop ^ 1)]) |*ctx_previous| { - defer self.events_contexts[@intCast(self.flip_flop ^ 1)] = null; - ctx_previous.event.waitUncancelable(self.io); - defer ctx_previous.pjrt_event.deinit(pjrt_api); - if (ctx_previous.err) |e| { - defer e.deinit(pjrt_api); - log.err("error while awaiting: {s}: {s}", .{ - @tagName(e.getCode(pjrt_api)), - e.getMessage(pjrt_api), - }); - return std.Io.Writer.Error.WriteFailed; - } - } - - if (is_last) { - defer ctx.* = null; - defer self.interface = .failing; - const ctx_ = &ctx.*.?; - ctx_.event.waitUncancelable(self.io); - defer ctx_.pjrt_event.deinit(pjrt_api); - if (ctx_.err) |e| { - defer e.deinit(pjrt_api); - log.err("error while awaiting: {s}: {s}", .{ - @tagName(e.getCode(pjrt_api)), - e.getMessage(pjrt_api), - }); - return std.Io.Writer.Error.WriteFailed; - } - } else { - self.interface.end = 0; - const buf = self.pool.get(self.allocator, self.io) catch |err| { - log.err("unable to get a new buffer from the pool: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - self.interface.buffer = buf[0..@min(buf.len, self.total - (self.offset + slice.len))]; - } - self.flip_flop ^= 1; - self.offset += slice.len; - } -}; - -const ShardProgress = struct { - const scale: usize = 1024; - - node: std.Progress.Node, - label: [32]u8 = undefined, - completed: usize = 0, - - fn set(self: *ShardProgress, completed: usize) void { - self.completed = completed; - self.node.setCompletedItems(std.math.divCeil(usize, self.completed, scale) catch unreachable); - } -}; - -// Dispatch planning bridges two different orders: -// -// 1. Placement traversal emits byte ranges per shard writer, in device order. -// That order is convenient for asking "what bytes belong to this shard?", -// but it does not match how a tensor file is read. -// -// 2. Readers stream tensor bytes in global row-major order. DirectMemoryWriter -// must therefore consume dispatch spans in increasing global byte offset. -// -// We first collect placement spans, sort them by global byte range, then fold -// identical ranges into one primary dispatch span plus mirror writers. Identical -// ranges represent replicated shard data: the reader provides those bytes once, -// the primary writer receives them zero-copy, and mirrors receive a copy. -const DispatchSpans = struct { - const DispatchSpan = struct { - start: usize, - end: usize, - primary_writer: usize, - mirror_writer_start: usize, - mirror_writer_len: usize, - }; - - const PlacementSpan = struct { - writer_index: usize, - start: usize, - len: usize, - order: usize, - }; - - spans: []DispatchSpan, - mirror_writers: []usize, - - fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { - const placement = try Sharding.Placement.init(&shape); - const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); - - var placement_span_count: usize = 0; - for (0..ordered_devices.len) |device_index| { - placement_span_count += placementSpanCount(shape, placement.slices(device_index, shape).constSlice()); - } - - var placement_spans: std.ArrayList(PlacementSpan) = try .initCapacity(allocator, placement_span_count); - defer placement_spans.deinit(allocator); - - const byte_strides = shape.computeByteStrides(); - - for (0..ordered_devices.len) |writer_index| { - appendShardPlacementSpans(&placement_spans, shape, placement.slices(writer_index, shape).constSlice(), byte_strides.constSlice(), writer_index); - } - - std.debug.assert(placement_spans.items.len == placement_span_count); - - var spans: std.ArrayList(DispatchSpan) = try .initCapacity(allocator, placement_spans.items.len); - errdefer spans.deinit(allocator); - - var mirror_writers: std.ArrayList(usize) = try .initCapacity(allocator, placement_spans.items.len); - errdefer mirror_writers.deinit(allocator); - - try deduplicateByRange(allocator, placement_spans.items, shape.byteSize(), &spans, &mirror_writers); - - const spans_ = try spans.toOwnedSlice(allocator); - errdefer allocator.free(spans_); - - const mirror_writers_ = try mirror_writers.toOwnedSlice(allocator); - errdefer allocator.free(mirror_writers_); - - return .{ - .spans = spans_, - .mirror_writers = mirror_writers_, - }; - } - - fn deinit(self: DispatchSpans, allocator: std.mem.Allocator) void { - allocator.free(self.spans); - allocator.free(self.mirror_writers); - } - - fn deduplicateByRange( - allocator: std.mem.Allocator, - placement_spans: []PlacementSpan, - total_bytes: usize, - spans: *std.ArrayList(DispatchSpan), - mirror_writers: *std.ArrayList(usize), - ) !void { - const SortContext = struct { - fn lessThan(_: void, lhs: PlacementSpan, rhs: PlacementSpan) bool { - if (lhs.start != rhs.start) return lhs.start < rhs.start; - if (lhs.len != rhs.len) return lhs.len < rhs.len; - return lhs.order < rhs.order; - } - }; - - std.mem.sort(PlacementSpan, placement_spans, {}, SortContext.lessThan); - - var i: usize = 0; - var cursor: usize = 0; - while (i < placement_spans.len) { - const span = placement_spans[i]; - if (span.start != cursor) return error.NonContiguousShardPlacement; - - const mirror_writer_start = mirror_writers.items.len; - var j = i + 1; - while (j < placement_spans.len) : (j += 1) { - const mirror = placement_spans[j]; - if (mirror.start != span.start or mirror.len != span.len) break; - try mirror_writers.append(allocator, mirror.writer_index); - } - - try spans.append(allocator, .{ - .start = span.start, - .end = span.start + span.len, - .primary_writer = span.writer_index, - .mirror_writer_start = mirror_writer_start, - .mirror_writer_len = j - i - 1, - }); - cursor += span.len; - i = j; - } - - if (cursor != total_bytes) return error.NonContiguousShardPlacement; - } - - fn appendPlacementSpan(placement_spans: *std.ArrayList(PlacementSpan), writer_index: usize, start: usize, len: usize) void { - placement_spans.appendAssumeCapacity(.{ - .writer_index = writer_index, - .start = start, - .len = len, - .order = placement_spans.items.len, - }); - } - - fn appendShardPlacementSpans( - placement_spans: *std.ArrayList(PlacementSpan), - shape: Shape, - slices: []const Placement.Slice1d, - byte_strides: []const i64, - writer_index: usize, - ) void { - if (shape.rank() == 0) { - appendPlacementSpan(placement_spans, writer_index, 0, shape.byteSize()); - return; - } - - appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, 0, contiguousSliceAxis(shape, slices), 0); - } - - fn appendShardAxisPlacementSpans( - placement_spans: *std.ArrayList(PlacementSpan), - slices: []const Placement.Slice1d, - byte_strides: []const i64, - writer_index: usize, - axis: usize, - contiguous_axis: usize, - base_start: i64, - ) void { - const slice = slices[axis]; - if (slice.size == 0) return; - - if (axis == contiguous_axis) { - const span_start: usize = @intCast(base_start + slice.start * byte_strides[axis]); - const span_len: usize = @intCast(slice.size * byte_strides[axis]); - appendPlacementSpan(placement_spans, writer_index, span_start, span_len); - return; - } - - var i: i64 = 0; - while (i < slice.size) : (i += 1) { - const child_start = base_start + (slice.start + i) * byte_strides[axis]; - appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, axis + 1, contiguous_axis, child_start); - } - } - - fn placementSpanCount(shape: Shape, slices: []const Placement.Slice1d) usize { - if (shape.rank() == 0) return 1; - - const contiguous_axis = contiguousSliceAxis(shape, slices); - var count: usize = 1; - for (slices[0..contiguous_axis]) |slice| { - count *= @intCast(slice.size); - } - return count; - } - - fn contiguousSliceAxis(shape: Shape, slices: []const Placement.Slice1d) usize { - var axis = shape.rank() - 1; - while (axis > 0) { - const slice = slices[axis]; - if (slice.start != 0 or slice.size != shape.dim(axis)) break; - axis -= 1; - } - return axis; - } -}; - -// Direct load writer state machine. -// -// std.Io.Reader -// | -// v -// DirectMemoryWriter.interface.buffer -// | aliases active primary DirectShardWriter DMA buffer -// v -// commitWindow() -// |-- primary: advance DirectShardWriter.interface.end -// |-- mirrors: copy committed bytes with writeAll() -// |-- boundaries: flush full shard buffers / chunk fence -// v -// DirectShardWriter.flushBuffered() -// | -// v -// PJRT AsyncHostToDeviceTransferManager -// -// The public writer never owns a host staging buffer: it exposes a visible -// prefix of a real shard DMA buffer. The visible prefix may extend beyond the -// current dispatch span to coalesce reader requests, but `commitWindow` must -// scatter those bytes before submitting or rotating the active buffer. The -// first active-primary segment is zero-copy; crossed spans and mirrors treat -// the active buffer as scratch and copy into their own shard writers. -// -// `byte_cursor` is the global tensor position. Shard writer `interface.end` is -// local to the current shard DMA buffer. -pub const DirectMemoryWriter = struct { - allocator: std.mem.Allocator, - shard_writers: []DirectShardWriter, - // Global stream-order spans produced from placement; this is the routing table for committed bytes. - dispatch_spans: DispatchSpans, - // Current entry in `dispatch_spans.spans`; advances whenever `byte_cursor` reaches that span end. - span_index: usize = 0, - // Global tensor byte offset already scattered into shard writers. - byte_cursor: usize = 0, - // Shard writer whose DMA buffer is currently exposed as `interface.buffer`. - active_writer_index: usize, - // Start offset of new reader bytes inside the active shard writer buffer. - window_start: usize, - // Logical maximum public alias window before forcing a cross-shard flush fence. - dma_chunk_size: usize, - shard_progress: ?[]ShardProgress = null, - interface: std.Io.Writer, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - pools: []mem.DynamicBufferPool, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - ) !DirectMemoryWriter { - var placed_shape = shape; - if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; - const placement = try Sharding.Placement.init(&placed_shape); - const mesh = placement.mesh().?; - const ordered_devices = mesh.devicesInCanonicalOrder(); - var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); - errdefer allocator.free(shard_writers); - - var initialized: usize = 0; - errdefer for (shard_writers[0..initialized]) |*writer| { - writer.deinit(); - }; - - var pjrt_buffers: Buffer.Shards = .empty; - for (ordered_devices, 0..) |device, i| { - const pool = &pools[device.id]; - const shard_dma_allocator = dma_allocators[device.id].allocator(); - const pjrt_mem = platform.devices[device.id].memory(memory).?; - - shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.localShape(placed_shape)); - initialized += 1; - - pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); - } - - buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); - - const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); - errdefer dispatch_spans.deinit(allocator); - - const first_span = dispatch_spans.spans[0]; - const first_writer = &shard_writers[first_span.primary_writer]; - const first_window = @min(dma_chunk_size, first_writer.interface.buffer.len); - - return .{ - .allocator = allocator, - .shard_writers = shard_writers, - .dispatch_spans = dispatch_spans, - .active_writer_index = first_span.primary_writer, - .window_start = first_writer.interface.end, - .dma_chunk_size = dma_chunk_size, - .interface = .{ - .buffer = first_writer.interface.buffer[0..first_window], - .end = 0, - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - }, - }, - }; - } - - pub fn deinit(self: *DirectMemoryWriter) void { - self.setProgress(null); - for (self.shard_writers) |*writer| { - writer.deinit(); - } - self.allocator.free(self.shard_writers); - self.dispatch_spans.deinit(self.allocator); - } - - pub fn setProgress(self: *DirectMemoryWriter, progress: ?*std.Progress.Node) void { - const parent = progress orelse { - const states = self.shard_progress orelse return; - for (states) |*s| { - s.node.end(); - } - self.allocator.free(states); - self.shard_progress = null; - return; - }; - - std.debug.assert(self.shard_progress == null); - const states = self.allocator.alloc(ShardProgress, self.shard_writers.len) catch return; - for (states, self.shard_writers, 0..) |*state, writer, i| { - state.completed = 0; - const label = std.fmt.bufPrint(&state.label, "shard[{d}]", .{i}) catch unreachable; - const total_items = std.math.divCeil(usize, writer.total, ShardProgress.scale) catch unreachable; - state.node = parent.start(label, total_items); - state.node.setCompletedItems(0); - } - self.shard_progress = states; - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - const chunk = data[0]; - try self.commitWindow(); - - const writable = self.interface.buffer.len - self.interface.end; - if (writable == 0) return std.Io.Writer.Error.WriteFailed; - - const n = @min(writable, chunk.len); - @memcpy(self.interface.buffer[self.interface.end..][0..n], chunk[0..n]); - self.interface.end += n; - try self.commitWindow(); - - return n; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - // Commit the active public window if one remains - if (self.span_index < self.dispatch_spans.spans.len) { - try self.commitWindow(); - } - - for (self.shard_writers, 0..) |*shard_writer, i| { - try shard_writer.interface.flush(); - if (self.shard_progress) |states| { - states[i].set(shard_writer.total); - } - } - - self.interface = .failing; - } - - fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - if (self.interface.buffer.len - self.interface.end >= capacity) return; - if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - - try self.commitWindow(); - if (self.interface.buffer.len - self.interface.end < capacity) return std.Io.Writer.Error.WriteFailed; - } - - // `interface.buffer` is a clipped alias of the active primary shard DMA - // buffer. Reader writes advance only `DirectMemoryWriter.interface.end`; - // this function scatters that visible range through dispatch spans. - // - // The leading segment for `active_writer_index` is already in the right - // DMA buffer and is committed by advancing that shard writer. Later spans - // use the same bytes as scratch and are copied before any full buffers are - // submitted or the public alias is rotated. - fn commitWindow(self: *DirectMemoryWriter) std.Io.Writer.Error!void { - // All dispatch spans are consumed; no shard DMA buffer remains valid to - // expose as the public alias until flush marks the writer as failing. - if (self.span_index >= self.dispatch_spans.spans.len) { - self.interface.buffer = &.{}; - self.interface.end = 0; - return; - } - - // Only bytes after `window_start` are new reader writes. Bytes before - // it may already be pending in the active shard writer. - const source = self.interface.buffer[self.window_start..self.interface.end]; - if (source.len == 0) return; - - // The visible alias window may cross many dispatch spans. Walk the - // global cursor and route each subrange to its primary and mirrors. - var consumed: usize = 0; - while (consumed < source.len) { - if (self.span_index >= self.dispatch_spans.spans.len) return std.Io.Writer.Error.WriteFailed; - - const span = self.dispatch_spans.spans[self.span_index]; - if (self.byte_cursor < span.start or self.byte_cursor >= span.end) return std.Io.Writer.Error.WriteFailed; - - const n = @min(source.len - consumed, span.end - self.byte_cursor); - const chunk = source[consumed..][0..n]; - - // The first span in the active alias is already in the primary DMA - // buffer. Later ranges use the active buffer as scratch and must be - // copied into their real destination writer. - if (span.primary_writer == self.active_writer_index and consumed == 0) { - self.shard_writers[span.primary_writer].interface.end += n; - } else { - const primary_writer = &self.shard_writers[span.primary_writer]; - if (span.primary_writer == self.active_writer_index) { - @memmove(primary_writer.interface.buffer[primary_writer.interface.end..][0..n], chunk); - primary_writer.interface.end += n; - } else { - try primary_writer.interface.writeAll(chunk); - } - } - if (self.shard_progress) |states| { - states[span.primary_writer].set(states[span.primary_writer].completed + n); - } - - const mirror_writer_end = span.mirror_writer_start + span.mirror_writer_len; - for (self.dispatch_spans.mirror_writers[span.mirror_writer_start..mirror_writer_end]) |mirror_writer_index| { - const mirror_writer = &self.shard_writers[mirror_writer_index]; - if (mirror_writer_index == self.active_writer_index) { - @memmove(mirror_writer.interface.buffer[mirror_writer.interface.end..][0..n], chunk); - mirror_writer.interface.end += n; - } else { - try mirror_writer.interface.writeAll(chunk); - } - if (self.shard_progress) |states| { - states[mirror_writer_index].set(states[mirror_writer_index].completed + n); - } - } - - self.byte_cursor += n; - consumed += n; - if (self.byte_cursor == span.end) { - self.span_index += 1; - } - } - - // All scratch copies are complete now, so full DMA buffers may be - // submitted without invalidating bytes still needed for scatter. - for (self.shard_writers) |*writer| { - if (writer.interface.end == writer.interface.buffer.len) { - try writer.interface.flush(); - } - } - - if (@mod(self.byte_cursor, self.dma_chunk_size) == 0) { - for (self.shard_writers) |*writer| { - try writer.interface.flush(); - } - } - - // The last committed window may finish the tensor exactly. Leave an - // empty public window so any further writes fail through std.Io. - if (self.span_index >= self.dispatch_spans.spans.len) { - self.interface.buffer = &.{}; - self.interface.end = 0; - return; - } - - // Rotate the public alias to the primary writer for the next stream - // span. `window_start` preserves that writer's already-buffered prefix - // so only newly advanced bytes are scattered on the next commit. - const next_span = self.dispatch_spans.spans[self.span_index]; - const next_writer = &self.shard_writers[next_span.primary_writer]; - self.active_writer_index = next_span.primary_writer; - self.window_start = next_writer.interface.end; - - // Publish the next public alias. Its backing memory is the next primary - // shard buffer, but the visible length is allowed to coalesce across - // upcoming spans until a buffer or tensor boundary. - const total = self.dispatch_spans.spans[self.dispatch_spans.spans.len - 1].end; - const buffer_remaining = next_writer.interface.buffer.len - self.window_start; - const chunk_offset = @mod(self.byte_cursor, self.dma_chunk_size); - const chunk_remaining = if (chunk_offset == 0) self.dma_chunk_size else self.dma_chunk_size - chunk_offset; - const tensor_remaining = total - self.byte_cursor; - const visible_remaining = @min(buffer_remaining, @min(chunk_remaining, tensor_remaining)); - if (visible_remaining == 0) return std.Io.Writer.Error.WriteFailed; - - self.interface.buffer = next_writer.interface.buffer[0 .. self.window_start + visible_remaining]; - self.interface.end = self.window_start; - } -}; - -const DirectMemoryWriterDeviceTest = struct { - const WriteMode = enum { - stream_remaining, - writable_slice_greedy, - }; - - pub const Scenario = struct { - name: []const u8, - create_options: CreateOptions, - shape: Shape, - logical_mesh: Sharding.LogicalMesh, - strategy: Sharding.Strategy, - write_mode: WriteMode = .stream_remaining, - writable_slice_min_len: usize = 128, - pool_chunks: usize = 4, - pool_chunk_size: usize = 1 << 20, - memory: Memory.Kind = .default, - }; - - allocator: std.mem.Allocator, - io: std.Io, - - fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { - var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; - defer platform.deinit(self.allocator, self.io); - - const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); - std.log.warn("mesh: {f}", .{mesh}); - try self.runDirectMemoryWriter( - platform, - scenario.shape.withPartitioning(&mesh, partitioning), - scenario.write_mode, - scenario.writable_slice_min_len, - scenario.pool_chunks, - scenario.pool_chunk_size, - scenario.memory, - ); - } - - fn runDirectMemoryWriter( - self: DirectMemoryWriterDeviceTest, - platform: *const Platform, - shape: Shape, - write_mode: WriteMode, - writable_slice_min_len: usize, - pool_chunks: usize, - pool_chunk_size: usize, - memory: Memory.Kind, - ) !void { - const slice = try Slice.alloc(self.allocator, shape); - defer slice.free(self.allocator); - - for (slice.items(f32), 0..) |*e, i| { - e.* = @as(f32, @floatFromInt(i)); - } - - const pool_count = platform.devices.len; - const dma_allocators = try self.allocator.alloc(mem.DmaAllocator, pool_count); - defer self.allocator.free(dma_allocators); - for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(self.allocator, device); - } - - const pools = try self.allocator.alloc(mem.DynamicBufferPool, pool_count); - defer self.allocator.free(pools); - for (pools) |*pool| { - pool.* = .init(pool_chunks, pool_chunk_size); - } - defer for (pools, 0..) |*pool, i| { - pool.deinit(dma_allocators[i].allocator()); - }; - - var written_buffer: Buffer = undefined; - var writer: DirectMemoryWriter = try .init( - self.allocator, - self.io, - platform, - pools, - dma_allocators, - pool_chunk_size, - shape, - &written_buffer, - memory, - ); - defer writer.deinit(); - defer written_buffer.deinit(); - - switch (write_mode) { - .stream_remaining => { - var reader: std.Io.Reader = .fixed(slice.constData()); - const streamed = try reader.streamRemaining(&writer.interface); - try std.testing.expectEqual(slice.constData().len, streamed); - }, - .writable_slice_greedy => { - var offset: usize = 0; - while (offset < slice.constData().len) { - const min_len = @max(@as(usize, 1), writable_slice_min_len); - const dest = try writer.interface.writableSliceGreedy(min_len); - const to_write = @min(dest.len, slice.constData().len - offset); - if (to_write == 0) return std.Io.Writer.Error.WriteFailed; - @memcpy(dest[0..to_write], slice.constData()[offset..][0..to_write]); - writer.interface.advance(to_write); - offset += to_write; - } - }, - } - - try writer.interface.flush(); - try written_buffer.await(self.io); - - var written_slice = try written_buffer.toSliceAlloc(self.allocator, self.io); - defer written_slice.free(self.allocator); - try std.testing.expectEqualSlices(u8, slice.constData(), written_slice.constData()); - } -}; - -test "DirectMemoryWriter: replicated tensor" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "replicated_auto", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), - .logical_mesh = .mesh(.{ .x = .high_bandwidth }), - .strategy = .parseBindings(.{ .x = .link_x }), - }, .{ .rows = .replicated, .cols = .replicated }); -} - -test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_auto", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - }, .{ .rows = .replicated, .cols = .model }); -} - -test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "batch_model_2d_torus", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), - .logical_mesh = .mesh(.{ - .batch = .low_bandwidth, - .model = .high_bandwidth, - }), - .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), - }, .{ .batch = .batch, .model = .model }); -} - -test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_folded_2d_torus", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_y }); - break :blk strategy; - }, - }, .{ .model = .model }); -} - -test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_auto_writable_slice", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - .write_mode = .writable_slice_greedy, - .writable_slice_min_len = 64, - .pool_chunk_size = 1024, - }, .{ .rows = .replicated, .cols = .model }); -} - -test "DirectMemoryWriter: 3D topology folded model + replicated batch" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "topology_3d_folded_model", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, - .cpu = .{ .device_count = 8 }, - }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ - .batch = .low_bandwidth, - .model = .high_bandwidth, - }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_z }); - break :blk strategy; - }, - }, .{ .batch = .replicated, .model = .model }); -} - -test "MemoryWriter can produce a host pinned buffer" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "host_pinned", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus1d }, - }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - .memory = .host_pinned, - }, .{ .batch = .replicated, .model = .model }); +test { + std.testing.refAllDecls(@This()); } diff --git a/zml/io/ShardGrid.zig b/zml/io/ShardGrid.zig new file mode 100644 index 00000000..41969285 --- /dev/null +++ b/zml/io/ShardGrid.zig @@ -0,0 +1,352 @@ +//! Where a destination's logical bytes land. Its devices hold the boxes of an +//! even grid over the tensor (`Sharding.Placement` splits every axis into +//! equal shards), each box packed row-major on its devices, and devices +//! holding the same box are one writer set. Box order and logical order agree +//! on a box's elements, so a contiguous logical range lands as one contiguous +//! range of every box it touches: ranges are placed by arithmetic, never by +//! listing rows. +//! +//! For a [4, 8] tensor whose columns are split in two, each letter one +//! element: +//! +//! Logical bytes Box 0 Box 1 +//! a b c d e f g h a b c d e f g h +//! i j k l m n o p i j k l m n o p +//! q r s t u v w x q r s t u v w x +//! y z A B C D E F y z A B C D E F +//! +//! The logical range from c to n lands as c..l on box 0 and e..n on box 1. +//! Runs are the aligned pieces that stay in one box (`run`: a shard of the +//! innermost split axis, four elements here); a worker walks them to place +//! a range's bytes, the planner only counts them. + +const std = @import("std"); + +const constants = @import("../constants.zig"); +const Platform = @import("../platform.zig").Platform; +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); + +const ShardGrid = @This(); + +const max_rank = constants.MAX_RANK; +/// Every box has at least one device, and a writer mask holds 64. +pub const max_boxes = 64; + +rank: usize, +/// Elements per axis, of the tensor and of one box. +dims: [max_rank]u64 = @splat(1), +shard: [max_rank]u64 = @splat(1), +/// Byte strides, logical and inside a box. +strides: [max_rank]u64 = @splat(0), +box_strides: [max_rank]u64 = @splat(0), +/// A box's index is row-major over the shard counts of the axes. +index_strides: [max_rank]u64 = @splat(0), +element: u64, +total: u64, +box_bytes: u64, +/// Aligned runs of `run` logical bytes each land in one box, contiguous. +run: u64, +box_count: usize, +/// The canonical device indices holding each box. +writers: [max_boxes]u64 = @splat(0), +/// One box: the shape each device holds. +local: Shape, +/// The devices in canonical order, which the `writers` masks index; +/// borrowed from the placed shape's mesh (the platform's when `shape` names +/// none). +devices: []const Sharding.Device, + +/// The grid of `shape` placed on `platform` (`Platform.placedShape`), its +/// sub-byte storage packed (`Shape.packedShape`): what PJRT allocates. +/// `shape` must be non-empty and place on the platform's devices, as the +/// loader's front end checked. +pub fn init(platform: *const Platform, shape: Shape) ShardGrid { + const placed = platform.placedShape(shape).packedShape(); + std.debug.assert(placed.byteSize() != 0); + const placement = Sharding.Placement.init(&placed) catch |err| + std.debug.panic("ShardGrid: {f} does not place, though checked: {s}", .{ placed, @errorName(err) }); + const local = placement.localShape(placed); + const devices = placement.mesh().?.devicesInCanonicalOrder(); + std.debug.assert(devices.len <= max_boxes); + const element: u64 = placed.dtype().sizeOf(); + var grid: ShardGrid = .{ + .rank = placed.rank(), + .element = element, + .total = placed.byteSize(), + .box_bytes = local.byteSize(), + .run = placed.byteSize(), + .box_count = 1, + .local = local, + .devices = devices, + }; + const strides = placed.computeByteStrides(); + var box_stride = element; + var index_stride: u64 = 1; + var axis = grid.rank; + while (axis > 0) { + axis -= 1; + const dim: u64 = @intCast(placed.dim(axis)); + const shard: u64 = @intCast(local.dim(axis)); + grid.dims[axis] = dim; + grid.shard[axis] = shard; + grid.strides[axis] = @intCast(strides.get(axis)); + grid.box_strides[axis] = box_stride; + grid.index_strides[axis] = index_stride; + box_stride *= shard; + const count = dim / shard; + index_stride *= count; + // The innermost split axis cuts the runs. + if (count > 1 and grid.run == grid.total) grid.run = shard * grid.strides[axis]; + } + // A placement holds every box on at least one of its devices. + std.debug.assert(index_stride <= devices.len); + grid.box_count = @intCast(index_stride); + for (0..devices.len) |writer| { + const slices = placement.slices(writer, placed); + var box: u64 = 0; + for (slices.constSlice(), 0..) |slice, i| box += @as(u64, @intCast(slice.start)) / grid.shard[i] * grid.index_strides[i]; + grid.writers[@intCast(box)] |= @as(u64, 1) << @intCast(writer); + } + for (grid.writers[0..grid.box_count]) |mask| std.debug.assert(mask != 0); + return grid; +} + +fn coordinate(self: *const ShardGrid, offset: u64, axis: usize) u64 { + return offset / self.strides[axis] % self.dims[axis]; +} + +/// The box holding logical byte `offset`, and where the byte lands inside it. +pub fn locate(self: *const ShardGrid, offset: u64) struct { usize, u64 } { + var box: u64 = 0; + var at = offset % self.element; + for (0..self.rank) |i| { + const c = self.coordinate(offset, i); + box += c / self.shard[i] * self.index_strides[i]; + at += c % self.shard[i] * self.box_strides[i]; + } + return .{ @intCast(box), at }; +} + +/// The bytes of box `box` before logical offset `offset`: a logical range +/// `[start, end)` lands on that box at `[rankIn(start), rankIn(end))`. +/// Counted axis by axis, as in a lexicographic comparison. +pub fn rankIn(self: *const ShardGrid, box: usize, offset: u64) u64 { + if (offset >= self.total) return self.box_bytes; + var before: u64 = 0; + for (0..self.rank) |i| { + const c = self.coordinate(offset, i); + const first = @as(u64, box) / self.index_strides[i] % (self.dims[i] / self.shard[i]) * self.shard[i]; + before += @min(c -| first, self.shard[i]) * self.box_strides[i]; + if (c < first or c >= first + self.shard[i]) return before; + } + return before + offset % self.element; +} + +/// The end of the run holding logical byte `offset`, at most `limit`. +pub fn runEnd(self: *const ShardGrid, offset: u64, limit: u64) u64 { + return @min(limit, (offset / self.run + 1) * self.run); +} + +/// The runs with a byte in `[start, end)`, `start < end`. +pub fn runCount(self: *const ShardGrid, start: u64, end: u64) u64 { + return (end + self.run - 1) / self.run - start / self.run; +} + +/// Devices per box: the grid is even, so every box has as many. +pub fn replicas(self: *const ShardGrid) usize { + return @popCount(self.writers[0]); +} + +const testing = struct { + const CreateOptions = @import("../platform.zig").CreateOptions; + + const Scenario = struct { + name: []const u8, + device_count: u32, + physical_mesh: CreateOptions.PhysicalMesh = .auto, + shape: Shape, + logical_mesh: Sharding.LogicalMesh, + strategy: Sharding.Strategy, + /// Logical ranges are cut every `step` bytes, unaligned to rows. + step: u64, + }; + + fn run(scenario: Scenario, partitioning: anytype) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + var platform = Platform.init(allocator, io, .cpu, .{ + .physical_mesh = scenario.physical_mesh, + .cpu = .{ .device_count = scenario.device_count }, + }) catch return error.SkipZigTest; + defer platform.deinit(allocator, io); + const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + try expectPlaces(allocator, platform, scenario.shape.withPartitioning(&mesh, partitioning), scenario.step); + } + + /// Walks the runs of every `step`-byte range into the writers' boxes and + /// compares with each device's slices, element by element; checks that + /// `rankIn` bounds the range's bytes on each box and that `runCount` + /// counts the runs. + fn expectPlaces(allocator: std.mem.Allocator, platform: *const Platform, shape: Shape, step: u64) !void { + const grid: ShardGrid = .init(platform, shape); + const placed = platform.placedShape(shape).packedShape(); + const placement: Sharding.Placement = try .init(&placed); + const box_bytes: usize = @intCast(grid.box_bytes); + const total: usize = @intCast(grid.total); + const source = try allocator.alloc(u8, total); + defer allocator.free(source); + for (source, 0..) |*byte, i| byte.* = @truncate(i *% 131 +% 17); + + const expected = try allocator.alloc(u8, grid.devices.len * box_bytes); + defer allocator.free(expected); + const element: usize = @intCast(grid.element); + for (0..grid.devices.len) |writer| { + const slices = placement.slices(writer, placed); + var index: [max_rank]u64 = @splat(0); + var cursor: usize = 0; + while (cursor < box_bytes) : (cursor += element) { + var logical: u64 = 0; + for (0..grid.rank) |i| logical += (@as(u64, @intCast(slices.get(i).start)) + index[i]) * grid.strides[i]; + @memcpy(expected[writer * box_bytes + cursor ..][0..element], source[@intCast(logical)..][0..element]); + var axis = grid.rank; + while (axis > 0) { + axis -= 1; + index[axis] += 1; + if (index[axis] < grid.shard[axis]) break; + index[axis] = 0; + } + } + } + + const actual = try allocator.alloc(u8, expected.len); + defer allocator.free(actual); + @memset(actual, 0); + var start: u64 = 0; + while (start < grid.total) : (start += step) { + const end = @min(grid.total, start + step); + var low: [max_boxes]u64 = @splat(std.math.maxInt(u64)); + var high: [max_boxes]u64 = @splat(0); + var runs: u64 = 0; + var at = start; + while (at < end) : (runs += 1) { + const stop = grid.runEnd(at, end); + const box, const landing = grid.locate(at); + low[box] = @min(low[box], landing); + high[box] = @max(high[box], landing + stop - at); + var mask = grid.writers[box]; + while (mask != 0) : (mask &= mask - 1) { + const writer: usize = @ctz(mask); + @memcpy(actual[writer * box_bytes + @as(usize, @intCast(landing)) ..][0..@intCast(stop - at)], source[@intCast(at)..@intCast(stop)]); + } + at = stop; + } + try std.testing.expectEqual(runs, grid.runCount(start, end)); + for (0..grid.box_count) |box| { + const first = grid.rankIn(box, start); + const last = grid.rankIn(box, end); + if (high[box] == 0) { + try std.testing.expectEqual(first, last); + } else { + try std.testing.expectEqual(low[box], first); + try std.testing.expectEqual(high[box], last); + } + } + } + try std.testing.expectEqualSlices(u8, expected, actual); + } +}; + +test "a grid places replicated tensors, across range boundaries" { + try testing.run(.{ + .name = "replicated", + .device_count = 4, + .shape = Shape.init(.{ .rows = 9, .cols = 257 }, .f32), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + .step = 773, + }, .{ .rows = .replicated, .cols = .replicated }); +} + +test "a grid places packed sub-byte storage" { + const logical = Shape.init(.{ .rows = 9, .cols = 256 }, .u2); + try testing.run(.{ + .name = "packed_u2", + .device_count = 4, + .shape = logical, + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + .step = 131, + }, .{ .rows = .replicated, .cols = .x }); +} + +test "a grid places 1D mirrored and folded sharding" { + try testing.run(.{ + .name = "mirrored_1d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .rows = 7, .model = 1024 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .step = 2053, + }, .{ .rows = .replicated, .model = .model }); + try testing.run(.{ + .name = "folded_1d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); + break :blk strategy; + }, + .step = 3001, + }, .{ .model = .model }); +} + +test "a grid places 2D and 3D sharding" { + try testing.run(.{ + .name = "batch_model_2d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), + .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), + .step = 4093, + }, .{ .batch = .batch, .model = .model }); + try testing.run(.{ + .name = "folded_model_3d", + .device_count = 8, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + break :blk strategy; + }, + .step = 8191, + }, .{ .batch = .replicated, .model = .model }); + // Three axes, the middle and the last split: runs are a quarter row. + try testing.run(.{ + .name = "middle_and_last_3d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .a = 3, .b = 4, .c = 8 }, .u16), + .logical_mesh = .mesh(.{ .x = .low_bandwidth, .y = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x, .y = .link_y }), + .step = 37, + }, .{ .a = .replicated, .b = .x, .c = .y }); +} + +test "a grid places a scalar on every device" { + try testing.run(.{ + .name = "scalar", + .device_count = 4, + .shape = Shape.init(.{}, .f32), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + .step = 3, + }, .{}); +} diff --git a/zml/io/Source.zig b/zml/io/Source.zig new file mode 100644 index 00000000..ea90111a --- /dev/null +++ b/zml/io/Source.zig @@ -0,0 +1,1349 @@ +//! A tensor the loader assembles from checkpoint bytes by moving them, never +//! by computing: a checkpoint tensor, or slices, reshapes, transposes, +//! broadcasts, paddings, concatenations and stacks of checkpoint tensors. +//! The ops are `zml.Tensor`'s shape ops, with the same names, tags and +//! argument conventions; slices are static and contiguous, and padding is +//! zeros only. +//! +//! A source comes from its store (`View.source`) and its pieces live in +//! the store's arena, where the ops that rewrite them allocate. Nothing is +//! freed before `TensorStore.deinit`, and the store must not move while +//! sources point to it. The ops touch neither the registry nor the +//! bindings. Sources of different stores do not mix: `concatenate`, +//! `stack` and `bindProgram` panic on them. Placement comes last: +//! `withPartitioning` and `as` return a `Placed`, which has no ops and +//! drops its placement with `unplaced`. A `Placed` is what binds: +//! `Placed.bind` records the model tensor the loader fills from it, +//! `bindProgram` one a program computes from several. Sources are built +//! during model init. +const std = @import("std"); + +const stdx = @import("stdx"); + +const constants = @import("../constants.zig"); +const DataType = @import("../dtype.zig").DataType; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; +const backend = @import("backend.zig"); +const TensorStore = @import("TensorStore.zig"); + +const Source = @This(); + +store: *TensorStore, +/// Always replicated: placement comes last (`Placed`). +_shape: Shape, +/// Disjoint lattices of elements read from checkpoint tensors; elements no +/// piece covers are zero. zml/io's own: a binding keeps them (`Placed`) +/// and the loader lowers them to bytes (`Placed.lower`). +_pieces: []const Piece, + +/// Checkpoint bits read through an affine lattice: for every `j` with +/// `j[k] < dims[k].count`, the element at row-major index +/// `origin + sum(j[k] * dims[k].step)` of the source's shape is the element +/// at bit `offset + sum(j[k] * dims[k].stride)` of `tensor`. Reshapes keep +/// the row-major order, so they never touch a piece. +const Piece = struct { + tensor: *const safetensors.Tensor, + offset: u64, + origin: u64, + /// Outermost first, none of count 1. + dims: []const Dim, +}; + +const Dim = struct { + count: u64, + /// Row-major elements of the source's shape between two steps. + step: u64, + /// Bits of the checkpoint tensor between two steps; zero repeats them. + stride: u64, +}; + +/// `pad`'s per-axis options, as for `Tensor.pad`. +pub const Pad = Tensor.Pad; + +/// The checkpoint tensor `subkey` of `view`, with `tags` (null keeps the +/// checkpoint's); null when the checkpoint has no such tensor. +pub fn init(view: TensorStore.View, subkey: []const u8, tags: ?[]const Shape.Tag) ?Source { + const tensor = view.get(subkey) orelse return null; + const store = view.store; + var shape_ = tensor.shape; + if (tags) |user_tags| { + stdx.debug.assert(user_tags.len == shape_.rank(), "checkpoint tensor {s} has shape {f}, but got {d} tags: {f}", .{ tensor.name, shape_, user_tags.len, stdx.fmt.stringsZ(user_tags) }); + @memcpy(shape_._tags.slice(), user_tags); + } + shape_._sharding = .replicated; + const arena = store.arena.allocator(); + const elements: u64 = shape_.count(); + const pieces: []const Piece = if (elements == 0) &.{} else blk: { + const dims_: []const Dim = if (elements == 1) + &.{} + else + arena.dupe(Dim, &.{.{ .count = elements, .step = 1, .stride = shape_.dtype().bitSizeOf() }}) catch oom(); + break :blk arena.dupe(Piece, &.{.{ .tensor = tensor, .offset = 0, .origin = 0, .dims = dims_ }}) catch oom(); + }; + return .{ .store = store, ._shape = shape_, ._pieces = pieces }; +} + +pub fn shape(self: Source) Shape { + return self._shape; +} + +pub fn dim(self: Source, axis_: anytype) i64 { + return self._shape.dim(axis_); +} + +pub fn dims(self: *const Source) []const i64 { + return self._shape.dims(); +} + +pub fn dtype(self: Source) DataType { + return self._shape.dtype(); +} + +pub fn rank(self: Source) u4 { + return self._shape.rank(); +} + +pub fn axis(self: Source, axis_: anytype) u3 { + return self._shape.axis(axis_); +} + +pub fn count(self: Source) usize { + return self._shape.count(); +} + +pub fn format(self: Source, writer: *std.Io.Writer) !void { + try writer.print("Source({f}, {d} pieces)", .{ self._shape, self._pieces.len }); +} + +// Reshapes: the row-major order is unchanged, so are the pieces. + +pub fn withTags(self: Source, tags: anytype) Source { + return self.reshaped(self._shape.withTags(tags)); +} + +pub fn rename(self: Source, renames: anytype) Source { + return self.reshaped(self._shape.rename(renames)); +} + +pub fn renameTag(self: Source, old: anytype, new: anytype) Source { + return self.reshaped(self._shape.setTag(old, new)); +} + +pub fn splitAxis(self: Source, ax: anytype, split_shape: anytype) Source { + return self.reshaped(self._shape.splitAxis(ax, split_shape)); +} + +pub fn merge(self: Source, merges: anytype) Source { + return self.reshaped(self._shape.mergeAxes(merges)); +} + +pub fn reshape(self: Source, output_shape: anytype) Source { + return self.reshaped(self._shape.reshape(output_shape)); +} + +pub fn flatten(self: Source) Source { + return self.reshape(.{self.count()}); +} + +pub fn insertAxes(self: Source, axis_: anytype, tags: anytype) Source { + const tags_ = Shape.parseTags(tags); + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); + var res = self._shape; + const ones: [constants.MAX_RANK]i64 = @splat(1); + res._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; + res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + return self.reshaped(res); +} + +pub fn appendAxes(self: Source, tags: anytype) Source { + return self.insertAxes(.last, tags); +} + +pub fn squeeze(self: Source, axis_: anytype) Source { + const a = self.axis(axis_); + stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); + return self.reshaped(self._shape.remove(a)); +} + +fn reshaped(self: Source, new_shape: Shape) Source { + stdx.debug.assert(new_shape.count() == self.count(), "Source reshape from {f} to {f} changes the element count", .{ self._shape, new_shape }); + var res = self; + res._shape = new_shape.withDtype(self.dtype()); + res._shape._sharding = .replicated; + return res; +} + +/// Reinterprets the bits as `dt`, like `Tensor.bitCast`: to a narrower type, +/// each element becomes the elements of a new last axis `.bitcast`; to a +/// wider one, the last axis must hold one element of `dt`. +pub fn bitCast(self: Source, dt: DataType) Source { + const src_bits: u64 = self.dtype().bitSizeOf(); + const tgt_bits: u64 = dt.bitSizeOf(); + if (src_bits == tgt_bits) return .{ .store = self.store, ._shape = self._shape.withDtype(dt), ._pieces = self._pieces }; + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + if (src_bits > tgt_bits) { + const ratio = std.math.divExact(u64, src_bits, tgt_bits) catch std.debug.panic("bitCast from {t} to {t}: bit sizes are not multiples", .{ self.dtype(), dt }); + const res_shape = self._shape.append(.{ .bitcast = @as(i64, @intCast(ratio)) }).withDtype(dt); + for (self._pieces) |piece| { + var flat: Flat = .fromPiece(piece); + flat.origin *= ratio; + for (flat.dims.slice()) |*d| d.step *= ratio; + flat.dims.append(.{ .count = ratio, .step = 1, .stride = tgt_bits }) catch tooManyDims(); + ctx.emitFlat(flat); + } + return ctx.finish(res_shape); + } + const ratio = std.math.divExact(u64, tgt_bits, src_bits) catch std.debug.panic("bitCast from {t} to {t}: bit sizes are not multiples", .{ self.dtype(), dt }); + const last: u3 = @intCast(self.rank() - 1); + stdx.debug.assert(self.dim(last) == @as(i64, @intCast(ratio)), "bitCast from {f} to {t} expects a last axis of {d} elements", .{ self._shape, dt, ratio }); + const res_shape = self._shape.remove(last).withDtype(dt); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |*axis_piece| { + // The elements merged into one must be one contiguous run of bits. + const inner: ?usize = for (axis_piece.dims.constSlice(), 0..) |d, k| { + if (d.axis == last) break k; + } else null; + const whole = axis_piece.origin[last] == 0 and inner != null and blk: { + const d = axis_piece.dims.get(inner.?); + break :blk d.count == ratio and d.step == 1 and d.stride == src_bits; + }; + if (!whole) std.debug.panic("bitCast of {f} to {t}: an element of {t} would gather bits that are not contiguous in the checkpoint", .{ self._shape, dt, dt }); + _ = axis_piece.dims.orderedRemove(inner.?); + ctx.emit(res_shape, axis_piece.*); + } + } + return ctx.finish(res_shape); +} + +// Ops that move elements. + +/// Like `Tensor.slice`, for a static, contiguous slice (`step` 1). +pub fn slice(self: Source, axis_: anytype, s: Tensor.Slice) Source { + stdx.debug.assert(s.dyn_start == null, "Source.slice takes static slices, got {f}", .{s}); + stdx.debug.assert(s.step == 1, "Source.slice takes contiguous slices, got {f}", .{s}); + const a = self.axis(axis_); + const range = s.absolute(self.dim(a)); + const lo: u64 = @intCast(range.start); + const hi: u64 = @intCast(range.end); + const res_shape = self._shape.setDim(a, range.end - range.start); + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |axis_piece| ctx.clip(res_shape, axis_piece, a, lo, hi); + } + const sliced = ctx.finish(res_shape); + return if (s.singleton) sliced.reshaped(res_shape.remove(a)) else sliced; +} + +/// Like `Tensor.split`; the result lives in the store's arena. +pub fn split(self: Source, axis_: anytype, split_sizes: []const i64) []Source { + const a = self.axis(axis_); + var total: i64 = 0; + for (split_sizes) |n| total += n; + stdx.debug.assert(total == self.dim(a), "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ total, self.dim(a) }); + const res = self.store.arena.allocator().alloc(Source, split_sizes.len) catch oom(); + var start: i64 = 0; + for (split_sizes, res) |n, *part| { + part.* = self.slice(a, .{ .start = start, .end = start + n }); + start += n; + } + return res; +} + +pub fn chunkExact(self: Source, axis_: anytype, comptime n_chunks: comptime_int) [n_chunks]Source { + const a = self.axis(axis_); + const size = @divExact(self.dim(a), n_chunks); + var chunks: [n_chunks]Source = undefined; + for (&chunks, 0..) |*chunk, i| { + const start = @as(i64, @intCast(i)) * size; + chunk.* = self.slice(a, .{ .start = start, .end = start + size }); + } + return chunks; +} + +/// Like `Tensor.transpose`: no axes reverses them. +pub fn transpose(self: Source, axes_: anytype) Source { + const given = self._shape.axes(axes_); + const r = self.rank(); + var permutation: [constants.MAX_RANK]u3 = undefined; + if (given.len == 0) { + for (0..r) |i| permutation[i] = @intCast(r - 1 - i); + } else { + stdx.debug.assert(given.len == r, "transpose expects {d} axes for {f}, got {d}", .{ r, self._shape, given.len }); + @memcpy(permutation[0..r], given.constSlice()); + } + var as_i64: [constants.MAX_RANK]i64 = undefined; + for (permutation[0..r], as_i64[0..r]) |p, *q| q.* = p; + const res_shape = self._shape.transpose(as_i64[0..r]); + // Axes of one element move nothing: when the others keep their order, + // the row-major order is unchanged, so are the pieces. + var previous: ?u3 = null; + const moves = for (permutation[0..r]) |p| { + if (self.dim(p) == 1) continue; + if (previous) |q| if (p < q) break true; + previous = p; + } else false; + if (!moves) return self.reshaped(res_shape); + var inverse: [constants.MAX_RANK]u3 = undefined; + for (permutation[0..r], 0..) |p, i| inverse[p] = @intCast(i); + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |axis_piece| { + var moved = axis_piece; + for (0..r) |i| moved.origin[i] = axis_piece.origin[permutation[i]]; + for (moved.dims.slice()) |*d| d.axis = inverse[d.axis]; + ctx.emit(res_shape, moved); + } + } + return ctx.finish(res_shape); +} + +/// Like `Tensor.pad`, with zero as the only padding value. +pub fn pad(self: Source, padding_value: anytype, paddings: anytype) Source { + stdx.debug.assert(isZero(padding_value), "Source.pad pads with zeros, got {any}", .{padding_value}); + const options = self._shape.parseAxesOptions(Tensor.Pad, paddings, .{}); + var res_shape = self._shape; + var moves = false; + for (options.constSlice(), 0..) |p, a| { + stdx.debug.assert(p.low >= 0 and p.high >= 0 and p.interior >= 0, "Source.pad cannot crop, got {any} on axis {d}", .{ p, a }); + const d = self.dim(a); + res_shape._dims.set(a, p.low + d + @max(d - 1, 0) * p.interior + p.high); + if (p.low != 0 or p.interior != 0 or (a != 0 and p.high != 0)) moves = true; + } + // Rows appended after the last one move no element. + if (!moves) return .{ .store = self.store, ._shape = res_shape, ._pieces = self._pieces }; + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |*axis_piece| { + for (options.constSlice(), 0..) |p, a| { + axis_piece.origin[a] = @as(u64, @intCast(p.low)) + axis_piece.origin[a] * @as(u64, @intCast(p.interior + 1)); + } + for (axis_piece.dims.slice()) |*d| d.step *= @intCast(options.get(d.axis).interior + 1); + ctx.emit(res_shape, axis_piece.*); + } + } + return ctx.finish(res_shape); +} + +/// Like `Tensor.broad`: repeats the elements along the axes `other` adds or +/// grows from 1. The loader reads a few large repetitions once each, and +/// copies many small ones from one read of the source per job. +pub fn broad(self: Source, other: Shape) Source { + stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self._shape, other }); + if (std.mem.eql(i64, self.dims(), other.dims())) return self; + const r = self.rank(); + var axes_map: [constants.MAX_RANK]u3 = undefined; + if (r == 0 or r == other.rank()) { + for (0..r) |i| axes_map[i] = @intCast(i); + } else { + for (self._shape.tags(), 0..) |t, i| axes_map[i] = other.axis(t); + } + const res_shape = other.withDtype(self.dtype()); + // The axes the elements repeat along: new ones and those grown from 1. + var repeated: [constants.MAX_RANK]bool = @splat(true); + for (0..r) |i| { + if (self.dim(i) != 1) repeated[axes_map[i]] = false; + } + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |axis_piece| { + var moved: AxisPiece = .{ .tensor = axis_piece.tensor, .offset = axis_piece.offset }; + for (0..r) |i| moved.origin[axes_map[i]] = axis_piece.origin[i]; + for (axis_piece.dims.constSlice()) |d| { + var mapped = d; + mapped.axis = axes_map[d.axis]; + moved.dims.append(mapped) catch tooManyDims(); + } + for (0..other.rank()) |b| { + if (!repeated[b] or other.dim(b) == 1) continue; + moved.dims.append(.{ .count = @intCast(other.dim(b)), .axis = @intCast(b), .step = 1, .stride = 0 }) catch tooManyDims(); + } + ctx.emit(res_shape, moved); + } + } + return ctx.finish(res_shape); +} + +/// Like `Tensor.concatenate`; the sources must share every dimension but +/// `axis_`, and the dtype. +pub fn concatenate(sources: []const Source, axis_: anytype) Source { + stdx.debug.assert(sources.len > 0, "concatenate expects at least one source", .{}); + if (sources.len == 1) return sources[0]; + const first = sources[0]; + const a = first.axis(axis_); + var total: i64 = 0; + for (sources) |s| { + stdx.debug.assert(s.store == first.store, "concatenate expects sources of one store", .{}); + const same = s.dtype() == first.dtype() and s.rank() == first.rank() and for (0..first.rank()) |i| { + if (i != a and s.dim(i) != first.dim(i)) break false; + } else true; + stdx.debug.assert(same, "concatenate along axis {d} expects matching shapes, got {f} and {f}", .{ a, first._shape, s._shape }); + total += s.dim(a); + } + const res_shape = first._shape.setDim(a, total); + const outer_stride = elementStrides(res_shape)[a]; + var ctx: Rewrite = .init(first.store); + defer ctx.deinit(); + var offset: u64 = 0; + for (sources) |s| { + for (s._pieces) |piece| { + if (a == 0) { + // Along the outermost axis, a source's elements follow the previous one's. + var flat: Flat = .fromPiece(piece); + flat.origin += offset * outer_stride; + ctx.emitFlat(flat); + continue; + } + for (ctx.decompose(s._shape, piece)) |*axis_piece| { + axis_piece.origin[a] += offset; + ctx.emit(res_shape, axis_piece.*); + } + } + offset += @intCast(s.dim(a)); + } + return ctx.finish(res_shape); +} + +/// Like `Tensor.stack`; the sources must have the same shape and tags. +pub fn stack(sources: []const Source, axis_: anytype, tag: anytype) Source { + stdx.debug.assert(sources.len > 0, "stack expects at least one source", .{}); + const first = sources[0]; + for (sources[1..]) |s| { + stdx.debug.assert(s.store == first.store, "stack expects sources of one store", .{}); + stdx.debug.assert(first._shape.eqlWithTags(s._shape), "stack expects source shapes to match, got {f} and {f}", .{ first._shape, s._shape }); + } + const a: u3 = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) @intCast(first.rank()) else first.axis(axis_); + const res_shape = first._shape.insertTag(axis_, @intCast(sources.len), tag); + const r = first.rank(); + var ctx: Rewrite = .init(first.store); + defer ctx.deinit(); + for (sources, 0..) |s, i| { + for (s._pieces) |piece| { + if (a == 0) { + var flat: Flat = .fromPiece(piece); + flat.origin += i * first.count(); + ctx.emitFlat(flat); + continue; + } + for (ctx.decompose(s._shape, piece)) |axis_piece| { + var moved = axis_piece; + std.mem.copyBackwards(u64, moved.origin[a + 1 .. r + 1], axis_piece.origin[a..r]); + moved.origin[a] = i; + for (moved.dims.slice()) |*d| { + if (d.axis >= a) d.axis += 1; + } + ctx.emit(res_shape, moved); + } + } + } + return ctx.finish(res_shape); +} + +/// This source placed on `mesh_` by `partitioning`: a struct keyed by the +/// source's tags (axes it does not name are replicated), or a +/// `Sharding.Partitioning`. `.replicated` places it on every device without +/// a mesh, and takes only `.{}`. Placement comes last: a `Placed` only +/// binds (`Placed.bind`, `bindProgram`) or drops its placement +/// (`Placed.unplaced`). +pub fn withPartitioning(self: Source, comptime mesh_: @EnumLiteral(), partitioning: anytype) Placed { + var placed = self._shape; + if (mesh_ == .replicated) { + const T = @TypeOf(partitioning); + if (comptime !(stdx.meta.isStruct(T) and std.meta.fieldNames(T).len == 0)) + @compileError("Source.withPartitioning(.replicated, ...) takes .{}: a replicated source has no partitioning, got " ++ @typeName(T)); + } else { + placed = self._shape.withPartitioning(self.store.mesh(mesh_), partitioning); + } + return .{ .store = self.store, .shape = placed, ._pieces = self._pieces }; +} + +/// This source in `placed`, a shape built elsewhere: its dimensions, tags +/// and sharding. Like `reshape`, it moves no element: `placed` has the +/// source's element count and dtype. Placement comes last, as for +/// `withPartitioning`. +pub fn as(self: Source, placed: Shape) Placed { + stdx.debug.assert(placed.dtype() == self.dtype() and placed.count() == self.count(), "Source {f} cannot be placed as {f}", .{ self._shape, placed }); + return .{ .store = self.store, .shape = placed, ._pieces = self._pieces }; +} + +/// Makes a model tensor of `output` that `program` computes while loading, +/// `program.forward(output, inputs)` over the placed sources `inputs` in this +/// order (`TensorStore.Program.init`). The loader lands each input as it +/// lands a bound source, placed by its shape, in default device memory; +/// compiles the program once for every tensor of an equal program over the +/// same placed shapes; runs it, and frees the inputs. `forward` returns a +/// tensor of `output`, placed alike, in default device memory. +pub fn bindProgram(inputs: []const Placed, output: Shape, program: anytype) Tensor { + stdx.debug.assert(inputs.len != 0, "Source.bindProgram: a program computing {f} needs an input", .{output}); + const store = inputs[0].store; + for (inputs) |input| { + stdx.debug.assert(input.store == store, "Source.bindProgram: the inputs of {f} come from different stores", .{output}); + } + const arena = store.arena.allocator(); + const kept = arena.dupe(Placed, inputs) catch oom(); + const erased = TensorStore.Program.init(arena, program) catch oom(); + return bindNew(store, output, .{ .compute = .{ .program = erased, .inputs = kept } }); +} + +/// A source with its placement (`withPartitioning`, `as`): the only thing +/// that binds, and what a binding keeps (`TensorStore.Binding`). +pub const Placed = struct { + store: *TensorStore, + /// The placed shape, whose row-major elements the pieces index. + shape: Shape, + /// See `Source._pieces`. + _pieces: []const Piece, + + /// Makes the model tensor the loader fills from this source, in default + /// memory unless `View.setMemory` names another. The binding keeps the + /// source as it is; the loader lowers it to bytes (`lower`). + pub fn bind(self: Placed) Tensor { + return bindNew(self.store, self.shape, .{ .load = .{ .source = self } }); + } + + /// This source without its placement, for code that takes a placed + /// source and reworks it by its placement (a layout per device), then + /// places the result again. + pub fn unplaced(self: Placed) Source { + var shape_ = self.shape; + shape_._sharding = .replicated; + return .{ .store = self.store, ._shape = shape_, ._pieces = self._pieces }; + } + + /// The checkpoint tensor this source is, whole and alone, if it is: one + /// piece that reads all of its bits in order. + pub fn whole(self: Placed) ?*const safetensors.Tensor { + if (self._pieces.len != 1) return null; + const piece = self._pieces[0]; + if (piece.offset != 0 or piece.origin != 0 or piece.tensor.byteSize() != self.shape.byteSize()) return null; + const contiguous = switch (piece.dims.len) { + 0 => self.shape.count() == 1, + 1 => piece.dims[0].count == self.shape.count() and piece.dims[0].step == 1 and piece.dims[0].stride == self.shape.dtype().bitSizeOf(), + else => false, + }; + return if (contiguous) piece.tensor else null; + } + + /// The bytes the loader moves, one part per piece in piece order, in + /// `arena`; none for a source that reads no bytes. The one rule from a + /// source's bits to bytes: each part's dimensions outermost first, the + /// innermost run that is contiguous on both sides as its width. + /// `error.SubBytePosition` when a run starts or ends inside a byte. + pub fn lower(self: Placed, arena: std.mem.Allocator) error{ OutOfMemory, SubBytePosition }![]backend.LoadPart { + const parts = try arena.alloc(backend.LoadPart, self._pieces.len); + for (parts, self._pieces) |*part, piece| part.* = try lowerPiece(arena, self.shape, piece); + return parts; + } +}; + +/// A new tensor of `shape_`, bound to `binding`: the one maker of +/// `store`'s bindings (`Placed.bind`, `bindProgram`), whose pieces are +/// disjoint by construction: the loader relies on that and does not check. +fn bindNew(store: *TensorStore, shape_: Shape, binding: TensorStore.Binding) Tensor { + const tensor: Tensor = .fromShape(shape_); + store.id_to_sources.putNoClobber(store.allocator, tensor.id, binding) catch oom(); + return tensor; +} + +/// `piece` of a source placed in `shape_` as one part (`Placed.lower`), its +/// dimensions allocated with `allocator`, none for one run. +fn lowerPiece(allocator: std.mem.Allocator, shape_: Shape, piece: Piece) error{ OutOfMemory, SubBytePosition }!backend.LoadPart { + const ebits: u64 = shape_.dtype().bitSizeOf(); + var lowered: stdx.BoundedArray(backend.Lattice.Dim, backend.max_part_dims) = .empty; + for (piece.dims) |d| lowered.appendAssumeCapacity(.{ .count = d.count, .src_stride = d.stride, .dst_stride = d.step * ebits }); + var width: u64 = ebits; + if (lowered.len != 0) { + const inner = lowered.get(lowered.len - 1); + if (inner.dst_stride == ebits and inner.src_stride == ebits) { + width = inner.count * ebits; + _ = lowered.pop(); + } + } + const dst_bits = piece.origin * ebits; + var aligned = piece.offset % 8 == 0 and dst_bits % 8 == 0 and width % 8 == 0; + for (lowered.slice()) |*d| { + aligned = aligned and d.src_stride % 8 == 0 and d.dst_stride % 8 == 0; + d.src_stride /= 8; + d.dst_stride /= 8; + } + if (!aligned) return error.SubBytePosition; + return .{ .source = piece.tensor, .lattice = .{ + .src_offset = piece.offset / 8, + .dst_offset = dst_bits / 8, + .width = width / 8, + .dims = if (lowered.len == 0) &.{} else try allocator.dupe(backend.Lattice.Dim, lowered.constSlice()), + } }; +} + +// The machinery. A piece is kept flat; an op that addresses axes rewrites it +// in coordinates of the input shape (`AxisPiece`, every dimension moving one +// axis), moves the coordinates, then flattens it in the result's shape. + +/// A piece while an op rewrites it, its dimensions inline. +const Flat = struct { + tensor: *const safetensors.Tensor, + offset: u64, + origin: u64, + dims: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty, + + fn fromPiece(piece: Piece) Flat { + var flat: Flat = .{ .tensor = piece.tensor, .offset = piece.offset, .origin = piece.origin }; + flat.dims.appendSlice(piece.dims) catch tooManyDims(); + return flat; + } + + /// The same lattice with dimension `k` cut to `n` steps from step `first`. + fn sub(self: Flat, k: usize, first: u64, n: u64) Flat { + var res = self; + const d = self.dims.get(k); + res.origin += first * d.step; + res.offset += first * d.stride; + res.dims.buffer[k].count = n; + return res; + } +}; + +/// A piece in coordinates of a shape: the element at +/// `origin + sum(j[k] * dims[k].step along dims[k].axis)`. +const AxisPiece = struct { + tensor: *const safetensors.Tensor, + offset: u64, + origin: [constants.MAX_RANK]u64 = @splat(0), + dims: stdx.BoundedArray(AxisDim, backend.max_part_dims) = .empty, +}; + +const AxisDim = struct { + count: u64, + axis: u3, + step: u64, + stride: u64, +}; + +/// The scratch of one op and the pieces it makes. +const Rewrite = struct { + store: *TensorStore, + scratch: std.heap.ArenaAllocator, + work: std.ArrayList(Flat) = .empty, + /// The current piece in coordinates of its shape (`decompose`). + axis_pieces: std.ArrayList(AxisPiece) = .empty, + out: std.ArrayList(Piece) = .empty, + + fn init(store: *TensorStore) Rewrite { + return .{ .store = store, .scratch = .init(store.allocator) }; + } + + fn deinit(self: *Rewrite) void { + self.scratch.deinit(); + } + + fn finish(self: *Rewrite, res_shape: Shape) Source { + var s = res_shape; + s._sharding = .replicated; + return .{ + .store = self.store, + ._shape = s, + ._pieces = self.store.arena.allocator().dupe(Piece, self.out.items) catch oom(), + }; + } + + /// `piece` in coordinates of `shape_`, cut where a dimension would carry + /// from one axis into the next; valid until the next call. + fn decompose(self: *Rewrite, shape_: Shape, piece: Piece) []AxisPiece { + const scratch = self.scratch.allocator(); + self.axis_pieces.clearRetainingCapacity(); + self.work.append(scratch, .fromPiece(piece)) catch oom(); + const strides = elementStrides(shape_); + while (self.work.pop()) |flat| { + switch (fitAxes(shape_, strides, flat)) { + .piece => |axis_piece| self.axis_pieces.append(scratch, axis_piece) catch oom(), + .overflow => |overflowing| self.cut(shape_, strides, flat, overflowing), + } + } + return self.axis_pieces.items; + } + + /// Splits `flat`, which carries past the end of axis `a`, into pieces + /// that do not: at the row boundaries of `a` when its longest dimension + /// there steps regularly through the rows, else one piece per step. + fn cut(self: *Rewrite, shape_: Shape, strides: [constants.MAX_RANK]u64, flat: Flat, a: u3) void { + const scratch = self.scratch.allocator(); + const n: u64 = @intCast(shape_.dim(a)); + const c = (flat.origin / strides[a]) % n; + var longest: ?usize = null; + var longest_span: u64 = 0; + var total: u64 = 0; + for (flat.dims.constSlice(), 0..) |d, k| { + if (d.count <= 1 or axisOfStep(shape_.rank(), strides, d.step) != a) continue; + const span = (d.count - 1) * (d.step / strides[a]); + total += span; + if (longest == null or span > longest_span) { + longest = k; + longest_span = span; + } + } + const k = longest.?; + const d = flat.dims.get(k); + const s = d.step / strides[a]; + const rest = total - longest_span; + if (n % s == 0 and c % s + rest < s) { + const per_row = n / s; + var j: u64 = 0; + if (c >= s) { + // A partial first row. + const head = @min((n - c + s - 1) / s, d.count); + self.work.append(scratch, flat.sub(k, 0, head)) catch oom(); + j = head; + } + const rows = (d.count - j) / per_row; + if (rows > 0) { + var body = flat.sub(k, j, per_row); + body.dims.append(.{ .count = rows, .step = d.step * per_row, .stride = d.stride * per_row }) catch tooManyDims(); + self.work.append(scratch, body) catch oom(); + j += rows * per_row; + } + if (j < d.count) self.work.append(scratch, flat.sub(k, j, d.count - j)) catch oom(); + return; + } + // An irregular lattice: one piece per step of that dimension. + for (0..d.count) |step_index| { + var single = flat.sub(k, step_index, 1); + _ = single.dims.orderedRemove(k); + self.work.append(scratch, single) catch oom(); + } + } + + /// Emits in `res_shape` the elements of `axis_piece` whose coordinate on + /// `a` is in `[lo, hi)`, at that coordinate minus `lo`. + fn clip(self: *Rewrite, res_shape: Shape, axis_piece: AxisPiece, a: u3, lo: u64, hi: u64) void { + var span: u64 = 0; + var outer: ?usize = null; + for (axis_piece.dims.constSlice(), 0..) |d, k| { + if (d.axis != a or d.count <= 1) continue; + span += (d.count - 1) * d.step; + if (outer == null or d.step > axis_piece.dims.get(outer.?).step) outer = k; + } + const o = axis_piece.origin[a]; + if (o >= lo and o + span < hi) { + var kept = axis_piece; + kept.origin[a] = o - lo; + self.emit(res_shape, kept); + return; + } + if (o + span < lo or o >= hi) return; + // Cut along the outermost dimension on `a`: the steps whose whole + // sub-lattice is inside stay one piece, the partial ones recurse. + const k = outer.?; + const d = axis_piece.dims.get(k); + const inner = span - (d.count - 1) * d.step; + const full_lo: u64 = if (o >= lo) 0 else std.math.divCeil(u64, lo - o, d.step) catch unreachable; + const full_hi: u64 = if (o + inner >= hi) 0 else @min(d.count, (hi - 1 - inner - o) / d.step + 1); + if (full_lo < full_hi) { + var kept = axis_piece; + kept.dims.buffer[k].count = full_hi - full_lo; + kept.origin[a] = o + full_lo * d.step - lo; + kept.offset += full_lo * d.stride; + self.emit(res_shape, kept); + } + const touch_lo: u64 = if (o + inner >= lo) 0 else std.math.divCeil(u64, lo - o - inner, d.step) catch unreachable; + const touch_hi: u64 = @min(d.count, (hi - 1 - o) / d.step + 1); + const partial = [_][2]u64{ + .{ touch_lo, @min(full_lo, touch_hi) }, + .{ @max(full_hi, touch_lo, full_lo), touch_hi }, + }; + for (partial) |range| { + var j = range[0]; + while (j < range[1]) : (j += 1) { + var part = axis_piece; + part.origin[a] = o + j * d.step; + part.offset += j * d.stride; + _ = part.dims.orderedRemove(k); + self.clip(res_shape, part, a, lo, hi); + } + } + } + + /// Flattens `axis_piece` in `res_shape` into the result. + fn emit(self: *Rewrite, res_shape: Shape, axis_piece: AxisPiece) void { + const strides = elementStrides(res_shape); + var flat: Flat = .{ .tensor = axis_piece.tensor, .offset = axis_piece.offset, .origin = 0 }; + for (0..res_shape.rank()) |b| flat.origin += axis_piece.origin[b] * strides[b]; + for (axis_piece.dims.constSlice()) |d| { + if (d.count <= 1) continue; + flat.dims.append(.{ .count = d.count, .step = d.step * strides[d.axis], .stride = d.stride }) catch tooManyDims(); + } + self.emitFlat(flat); + } + + /// Adds `flat` to the result, its dimensions outermost first and merged + /// where one continues the other. + fn emitFlat(self: *Rewrite, flat: Flat) void { + var sorted: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty; + for (flat.dims.constSlice()) |d| { + if (d.count > 1) sorted.appendAssumeCapacity(d); + } + std.mem.sort(Dim, sorted.slice(), {}, struct { + fn moreOuter(_: void, lhs: Dim, rhs: Dim) bool { + return lhs.step > rhs.step; + } + }.moreOuter); + var merged: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty; + for (sorted.constSlice()) |d| { + if (merged.len != 0) { + const last = &merged.slice()[merged.len - 1]; + if (last.step == d.count * d.step and last.stride == d.count * d.stride) { + last.* = .{ .count = last.count * d.count, .step = d.step, .stride = d.stride }; + continue; + } + } + merged.appendAssumeCapacity(d); + } + const arena = self.store.arena.allocator(); + self.out.append(self.scratch.allocator(), .{ + .tensor = flat.tensor, + .offset = flat.offset, + .origin = flat.origin, + .dims = if (merged.len == 0) &.{} else arena.dupe(Dim, merged.constSlice()) catch oom(), + }) catch oom(); + } +}; + +const AxisFit = union(enum) { + piece: AxisPiece, + /// The axis some coordinate would run past. + overflow: u3, +}; + +/// `flat` in coordinates of `shape_`, if no dimension carries into the next +/// axis: each dimension moves the outermost axis its step is a multiple of. +fn fitAxes(shape_: Shape, strides: [constants.MAX_RANK]u64, flat: Flat) AxisFit { + const r = shape_.rank(); + var axis_piece: AxisPiece = .{ .tensor = flat.tensor, .offset = flat.offset }; + for (0..r) |b| axis_piece.origin[b] = (flat.origin / strides[b]) % @as(u64, @intCast(shape_.dim(b))); + var extent = axis_piece.origin; + for (flat.dims.constSlice()) |d| { + if (d.count <= 1) continue; + const b = axisOfStep(r, strides, d.step); + const step = d.step / strides[b]; + axis_piece.dims.appendAssumeCapacity(.{ .count = d.count, .axis = b, .step = step, .stride = d.stride }); + extent[b] += (d.count - 1) * step; + } + for (0..r) |b| { + if (extent[b] >= @as(u64, @intCast(shape_.dim(b)))) return .{ .overflow = @intCast(b) }; + } + return .{ .piece = axis_piece }; +} + +fn axisOfStep(r: u4, strides: [constants.MAX_RANK]u64, step: u64) u3 { + for (0..r) |b| { + if (step % strides[b] == 0) return @intCast(b); + } + std.debug.panic("a lattice step of {d} elements in a shape of rank {d}", .{ step, r }); +} + +/// Row-major element strides. +fn elementStrides(shape_: Shape) [constants.MAX_RANK]u64 { + var strides: [constants.MAX_RANK]u64 = @splat(0); + var acc: u64 = 1; + var b: usize = shape_.rank(); + while (b > 0) { + b -= 1; + strides[b] = acc; + acc *= @intCast(shape_.dim(b)); + } + return strides; +} + +fn isZero(value: anytype) bool { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_int, .int, .comptime_float, .float => value == 0, + else => false, + }; +} + +fn oom() noreturn { + @panic("OOM"); +} + +fn tooManyDims() noreturn { + std.debug.panic("a Source piece needs more than {d} dimensions", .{backend.max_part_dims}); +} + +// Tests: every op is checked against the same function run as a program +// over `zml.Tensor`, the loader's byte placement replayed on the host. + +const testing = @import("../testing.zig"); +const Buffer = @import("../buffer.zig").Buffer; + +/// A checkpoint of in-memory tensors (the registry points nowhere) and the +/// bytes each holds. +const TestCheckpoint = struct { + store: TensorStore, + bytes: std.StringHashMapUnmanaged([]u8) = .empty, + allocator: std.mem.Allocator, + + const Entry = struct { name: []const u8, shape: Shape }; + + /// Registers `entries` end to end, each holding distinct bytes. + fn init(self: *TestCheckpoint, allocator: std.mem.Allocator, entries: []const Entry) !void { + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var offset: u64 = 0; + for (entries) |entry| { + try registry.registerTensor(.{ .file_uri = "memory", .name = entry.name, .shape = entry.shape, .offset = offset }); + offset += entry.shape.byteSize(); + } + self.* = .{ .store = .init(allocator, ®istry, .{}), .allocator = allocator }; + errdefer self.deinit(); + for (entries, 0..) |entry, index| { + const bytes = try allocator.alloc(u8, entry.shape.byteSize()); + errdefer allocator.free(bytes); + for (bytes, 0..) |*byte, i| byte.* = @truncate(i *% 7 +% index *% 61 +% 1); + try self.bytes.put(allocator, self.store.registry.tensors.getKey(entry.name).?, bytes); + } + } + + fn deinit(self: *TestCheckpoint) void { + var it = self.bytes.valueIterator(); + while (it.next()) |bytes| self.allocator.free(bytes.*); + self.bytes.deinit(self.allocator); + self.store.deinit(); + } + + /// The bytes the loader would assemble for `tensor`, placed on the host: + /// its source lowered as the loader lowers it (`Placed.lower`), the + /// bytes no part covers zero. Fails when a part writes a byte another + /// wrote, so that every byte is written exactly once, by a part or the + /// zero fill: the loader relies on disjoint pieces and does not check + /// them. + fn assemble(self: *TestCheckpoint, tensor: Tensor) ![]u8 { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + const parts = try self.store.bindingOf(tensor.id).?.load.source.lower(arena.allocator()); + const out = try self.allocator.alloc(u8, tensor.shape().byteSize()); + errdefer self.allocator.free(out); + @memset(out, 0); + const written = try arena.allocator().alloc(bool, out.len); + @memset(written, false); + for (parts) |part| { + const src = self.bytes.get(part.source.name).?; + const width: usize = @intCast(part.lattice.width); + var runs: backend.Lattice.Cursor = .init(part.lattice, 0, part.lattice.runs()); + while (runs.next()) |run| { + const at: usize = @intCast(run.dst); + for (written[at..][0..width]) |*byte| { + if (byte.*) return error.DoubleWrite; + byte.* = true; + } + @memcpy(out[at..][0..width], src[@intCast(run.src)..][0..width]); + } + } + return out; + } +}; + +fn interleave(comptime T: type, members: []const T, groups: i64) T { + var parts: [4]T = undefined; + for (members, parts[0..members.len]) |member, *part| part.* = member.splitAxis(.row, .{ .group = groups, .r = -1 }); + return T.concatenate(parts[0..members.len], .r).merge(.{ .row = .{ .group, .r } }); +} + +const qkv_inputs = .{ .{ .row = 4, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 } }; +const expert_inputs = .{ .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 } }; + +/// `build(T, x)` builds a tensor from `x` with shape ops only, the same code +/// for `Tensor` and `Source`; `inputs` holds the dims of each `u16` input. +fn expectCase(comptime inputs: anytype, comptime build: anytype) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const n = inputs.len; + var names: [n][]const u8 = undefined; + var name_buffers: [n][16]u8 = undefined; + var shapes: [n]Shape = undefined; + var entries: [n]TestCheckpoint.Entry = undefined; + inline for (inputs, 0..) |dims_, i| { + names[i] = try std.fmt.bufPrint(&name_buffers[i], "input{d}", .{i}); + shapes[i] = Shape.init(dims_, .u16); + entries[i] = .{ .name = names[i], .shape = shapes[i] }; + } + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(allocator, &entries); + defer checkpoint.deinit(); + + var sources: [n]Source = undefined; + for (&sources, names) |*s, input_name| s.* = checkpoint.store.view().source(input_name, null); + const bound = build(Source, &sources).withPartitioning(.replicated, .{}).bind(); + const assembled = try checkpoint.assemble(bound); + defer allocator.free(assembled); + + const platform = testing.env(); + var tensors: [n]Tensor = undefined; + for (&tensors, shapes) |*t, shape_| t.* = .fromShape(shape_); + const Program = struct { + fn forward(x: []const Tensor) Tensor { + return build(Tensor, x); + } + }; + var exe = try platform.compileFn(allocator, io, Program.forward, .{&tensors}, .{}); + defer exe.deinit(); + var buffers: [n]Buffer = undefined; + var created: usize = 0; + defer for (buffers[0..created]) |*buffer| buffer.deinit(); + for (&buffers, shapes, names) |*buffer, shape_, input_name| { + buffer.* = try .fromBytes(io, platform, shape_, checkpoint.bytes.get(input_name).?); + created += 1; + } + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + args.set(.{buffers[0..]}); + try exe.call(io, args, &results, .{ .wait = true }); + var output = results.get(Buffer); + defer output.deinit(); + const expected = try output.toSliceAlloc(allocator, io); + defer expected.free(allocator); + + std.testing.expect(bound.shape().eqlWithTags(output.shape())) catch |err| { + std.log.err("Source gives {f}, Tensor {f}", .{ bound.shape(), output.shape() }); + return err; + }; + std.testing.expectEqualSlices(u8, expected.constData(), assembled) catch |err| { + std.log.err("the loader's bytes differ from the program's", .{}); + return err; + }; +} + +test "Source concatenates rows as Tensor does" { + try expectCase(.{ .{ .row = 3, .col = 4 }, .{ .row = 2, .col = 4 }, .{ .row = 4, .col = 4 } }, struct { + fn build(comptime T: type, x: []const T) T { + return T.concatenate(x, .row); + } + }.build); +} + +test "Source concatenates columns as Tensor does" { + try expectCase(.{ .{ .row = 3, .col = 4 }, .{ .row = 3, .col = 2 } }, struct { + fn build(comptime T: type, x: []const T) T { + return T.concatenate(x, .col); + } + }.build); +} + +test "Source interleaves q, k and v by heads as Tensor does" { + try expectCase(qkv_inputs, struct { + fn build(comptime T: type, x: []const T) T { + return interleave(T, x, 2); + } + }.build); +} + +test "Source interleaves rows as Tensor does" { + try expectCase(.{ .{ .row = 4, .col = 3 }, .{ .row = 4, .col = 3 } }, struct { + fn build(comptime T: type, x: []const T) T { + return interleave(T, x, 4); + } + }.build); +} + +test "Source stacks experts as Tensor does" { + try expectCase(expert_inputs, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; + return T.concatenate(&members, .row); + } + }.build); +} + +test "Source interleaves stacked experts as Tensor does" { + try expectCase(expert_inputs, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; + return T.stack(&members, 2, .proj).merge(.{ .rows = .{ .row, .proj } }); + } + }.build); +} + +test "Source interleaves expert vectors as Tensor does" { + // Per-expert vectors (biases): row r of projection p lands at r * 2 + p. + try expectCase(.{ .{ .row = 3 }, .{ .row = 3 }, .{ .row = 3 }, .{ .row = 3 } }, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; + return T.stack(&members, .last, .proj).merge(.{ .rows = .{ .row, .proj } }); + } + }.build); +} + +test "Source transposes as Tensor does" { + try expectCase(.{.{ .row = 3, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].transpose(.{ .col, .row }); + } + }.build); +} + +test "Source interleaves rows of three as Tensor does" { + // lfm2's in_proj: rows [B; C; x] become b0 c0 x0 b1 c1 x1. + try expectCase(.{.{ .row = 6, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].splitAxis(.row, .{ .m = 3, .c = 2 }).transpose(.{ .c, .m, .col }).merge(.{ .row = .{ .c, .m } }); + } + }.build); +} + +test "Source slices rows as Tensor does" { + try expectCase(.{.{ .row = 5, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.row, .{ .start = 1, .end = 4 }); + } + }.build); +} + +test "Source slices columns as Tensor does" { + try expectCase(.{.{ .row = 3, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.col, .{ .start = 1, .end = 3 }); + } + }.build); +} + +test "Source slices by length as Tensor does" { + try expectCase(.{.{ .row = 6, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.row, .{ .start = 2, .len = 3 }); + } + }.build); +} + +test "Source slices an interleave as Tensor does" { + try expectCase(qkv_inputs, struct { + fn build(comptime T: type, x: []const T) T { + return interleave(T, x, 2).slice(.row, .{ .start = 3, .end = 7 }); + } + }.build); +} + +test "Source takes a single row as Tensor does" { + try expectCase(.{.{ .row = 5, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.row, .single(2)); + } + }.build); +} + +test "Source pads as Tensor does" { + try expectCase(.{.{ .row = 4, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].pad(0, .{ .row = Tensor.Pad{ .high = 4 }, .col = Tensor.Pad{ .low = 1, .interior = 1 } }); + } + }.build); +} + +test "Source pads rows as Tensor does" { + try expectCase(.{.{ .row = 5, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].pad(0, .{ .row = Tensor.Pad{ .high = 3 } }); + } + }.build); +} + +test "Source broadcasts a scalar as Tensor does" { + try expectCase(.{.{}}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].broad(Shape.init(.{ .row = 3, .col = 2 }, .u16)); + } + }.build); +} + +test "Source broadcasts a column as Tensor does" { + // A per-tensor scale repeated over each member's rows. + try expectCase(.{ .{ .row = 1, .col = 1 }, .{ .row = 1, .col = 1 } }, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ x[0].broad(Shape.init(.{ .row = 4, .col = 1 }, .u16)), x[1].broad(Shape.init(.{ .row = 2, .col = 1 }, .u16)) }; + return interleave(T, &members, 2); + } + }.build); +} + +test "Source bitcasts to a narrower type as Tensor does" { + try expectCase(.{.{ .row = 3, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].bitCast(.u8).merge(.{ .col = .{ .col, .bitcast } }); + } + }.build); +} + +test "Source bitcasts to a wider type as Tensor does" { + try expectCase(.{.{ .row = 2, .col = 6 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].bitCast(.u8).splitAxis(.col, .{ .col = 3, .pair = 2 }).merge(.{ .pair = .{ .pair, .bitcast } }).bitCast(.u32); + } + }.build); +} + +test "Source swizzles block scales as Tensor does" { + // The 128x4 block-scale layout, at 8x4: rows padded to 8 and tiled + // by 2x4, columns padded to 4. + try expectCase(.{.{ .e = 2, .m = 7, .k = 6 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].pad(0, .{ .m = Tensor.Pad{ .high = 1 }, .k = Tensor.Pad{ .high = 2 } }) + .splitAxis(.m, .{ .mo = -1, .m4 = 2, .mi = 4 }) + .splitAxis(.k, .{ .ko = -1, .ki = 4 }) + .transpose(.{ .e, .mo, .ko, .mi, .m4, .ki }) + .merge(.{ .flat = .{ .mo, .ko, .mi, .m4, .ki } }); + } + }.build); +} + +test "Source splits then interleaves as Tensor does" { + // qwen3_5's in_proj_qkv sliced into q, k and v, interleaved with z. + try expectCase(.{ .{ .row = 6, .col = 3 }, .{ .row = 2, .col = 3 } }, struct { + fn build(comptime T: type, x: []const T) T { + const qkv = x[0]; + const members = [4]T{ + qkv.slice(.row, .{ .start = 0, .end = 2 }), + qkv.slice(.row, .{ .start = 2, .end = 4 }), + qkv.slice(.row, .{ .start = 4, .end = 6 }), + x[1], + }; + return interleave(T, &members, 2); + } + }.build); +} + +test "Source chunks then stacks as Tensor does" { + try expectCase(.{.{ .row = 6, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + const chunks = x[0].chunkExact(.row, 3); + return T.stack(&.{ chunks[2], chunks[0] }, .last, .pair).insertAxes(0, .{.one}).squeeze(0); + } + }.build); +} + +test "a source binds as its pieces, which the loader lowers to byte parts" { + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(std.testing.allocator, &.{ + .{ .name = "q", .shape = .init(.{ 4, 3 }, .u16) }, + .{ .name = "k", .shape = .init(.{ 2, 3 }, .u16) }, + .{ .name = "v", .shape = .init(.{ 2, 3 }, .u16) }, + }); + defer checkpoint.deinit(); + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const view = checkpoint.store.view(); + const q = view.source("q", .{ .row, .col }); + const entries = [_]*const safetensors.Tensor{ view.get("q").?, view.get("k").?, view.get("v").? }; + + // Relabels keep the whole tensor, one run of it; only `bind` records + // anything, and the binding keeps the placed source as it is. + const relabeled = q.splitAxis(.row, .{ .h = 2, .hd = 2 }).merge(.{ .row = .{ .h, .hd } }).withPartitioning(.replicated, .{}); + try std.testing.expectEqual(@as(usize, 0), checkpoint.store.id_to_sources.count()); + const bound = checkpoint.store.bindingOf(relabeled.bind().id).?.load.source; + try std.testing.expectEqual(relabeled._pieces, bound._pieces); + try std.testing.expectEqual(entries[0], bound.whole().?); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{.whole(entries[0])}, try bound.lower(arena.allocator())); + // One checkpoint tensor read twice is two pieces of it. + const twice = Source.concatenate(&.{ q, view.source("q", .{ .row, .col }) }, .row).withPartitioning(.replicated, .{}); + try std.testing.expectEqual(null, twice.whole()); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{ + .{ .source = entries[0], .lattice = .{ .dst_offset = 0, .width = 24 } }, + .{ .source = entries[0], .lattice = .{ .dst_offset = 24, .width = 24 } }, + }, try twice.lower(arena.allocator())); + + // q, k and v in two groups: each member is one part of two runs, its + // rows of a group side by side with the others'. + const members = [_]Source{ q, view.source("k", .{ .row, .col }), view.source("v", .{ .row, .col }) }; + const fused = interleave(Source, &members, 2).withPartitioning(.replicated, .{}); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{ + .{ .source = entries[0], .lattice = .{ .src_offset = 0, .dst_offset = 0, .width = 12, .dims = &.{.{ .count = 2, .src_stride = 12, .dst_stride = 24 }} } }, + .{ .source = entries[1], .lattice = .{ .src_offset = 0, .dst_offset = 12, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, + .{ .source = entries[2], .lattice = .{ .src_offset = 0, .dst_offset = 18, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, + }, try fused.lower(arena.allocator())); + + // A transpose is one part of single elements. + const transposed = q.transpose(.{}).withPartitioning(.replicated, .{}); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{ + .{ .source = entries[0], .lattice = .{ .src_offset = 0, .dst_offset = 0, .width = 2, .dims = &.{ + .{ .count = 3, .src_stride = 2, .dst_stride = 8 }, + .{ .count = 4, .src_stride = 6, .dst_stride = 2 }, + } } }, + }, try transposed.lower(arena.allocator())); +} + +test "assembling fails on a byte two pieces write" { + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(std.testing.allocator, &.{.{ .name = "q", .shape = .init(.{ 4, 3 }, .u16) }}); + defer checkpoint.deinit(); + // Only a hand-built binding can repeat a piece. + const q = checkpoint.store.view().source("q", .{ .row, .col }); + const pieces = try checkpoint.store.arena.allocator().dupe(Piece, &.{ q._pieces[0], q._pieces[0] }); + const tensor = bindNew(&checkpoint.store, q.shape(), .{ .load = .{ .source = .{ .store = &checkpoint.store, .shape = q.shape(), ._pieces = pieces } } }); + try std.testing.expectError(error.DoubleWrite, checkpoint.assemble(tensor)); +} + +test "a source is placed last, by as or withPartitioning, and binds in its placed shape" { + const allocator = std.testing.allocator; + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(allocator, &.{.{ .name = "w", .shape = .init(.{ 4, 6 }, .u16) }}); + defer checkpoint.deinit(); + const mesh: Sharding.Mesh = .initTest(.tp, .{.model}); + const transposed = checkpoint.store.view().source("w", .{ .row, .col }).transpose(.{}); + + // `as` takes a shape built elsewhere, here of another rank: its + // dimensions, tags and sharding, for the same elements. + const placed = Shape.init(.{ .flat = 24 }, .u16).withPartitioning(&mesh, .{ .flat = .model }); + const placed_source = transposed.as(placed); + const flat = placed_source.bind(); + try std.testing.expect(flat.shape().eqlWithTagsAndPartitioning(placed)); + try std.testing.expect(checkpoint.store.bindingOf(flat.id).?.load.source.shape.eqlWithTagsAndPartitioning(placed)); + const replicated = transposed.withPartitioning(.replicated, .{}).bind(); + try std.testing.expect(replicated.shape().eqlWithTags(transposed.shape())); + try std.testing.expectEqual(null, replicated.shape()._sharding.mesh); + // `unplaced` keeps the placed dimensions and tags, without the sharding. + const dropped = placed_source.unplaced(); + try std.testing.expect(dropped.shape().eqlWithTags(placed)); + try std.testing.expectEqual(null, dropped.shape()._sharding.mesh); + + const flat_bytes = try checkpoint.assemble(flat); + defer allocator.free(flat_bytes); + const replicated_bytes = try checkpoint.assemble(replicated); + defer allocator.free(replicated_bytes); + try std.testing.expectEqualSlices(u8, replicated_bytes, flat_bytes); +} + +test "lattices stay one piece per source at checkpoint sizes" { + // Registered without bytes: nothing is evaluated. + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "memory", .name = "gate", .shape = .init(.{ 14336, 4096 }, .bf16), .offset = 0 }); + try registry.registerTensor(.{ .file_uri = "memory", .name = "up", .shape = .init(.{ 14336, 4096 }, .bf16), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const view = store.view(); + const members = [_]Source{ view.source("gate", .{ .row, .col }), view.source("up", .{ .row, .col }) }; + + // One row per group: two pieces, each 14336 runs of one row. + const finest = interleave(Source, &members, 14336); + try std.testing.expectEqual(@as(usize, 2), finest._pieces.len); + for (try finest.withPartitioning(.replicated, .{}).lower(arena.allocator())) |part| { + try std.testing.expectEqual(@as(u64, 14336), part.lattice.runs()); + try std.testing.expectEqual(@as(u64, 8192), part.lattice.width); + } + // One group per device of four: four runs of 28 MiB each. + const per_shard = interleave(Source, &members, 4); + for (try per_shard.withPartitioning(.replicated, .{}).lower(arena.allocator())) |part| { + try std.testing.expectEqual(@as(u64, 4), part.lattice.runs()); + try std.testing.expectEqual(@as(u64, 3584 * 8192), part.lattice.width); + } + // Slicing through the interleave cuts at most a few pieces. + const sliced = finest.slice(.row, .{ .start = 1001, .end = 20001 }); + try std.testing.expect(sliced._pieces.len <= 6); +} diff --git a/zml/io/TensorStore.zig b/zml/io/TensorStore.zig new file mode 100644 index 00000000..3b159bbc --- /dev/null +++ b/zml/io/TensorStore.zig @@ -0,0 +1,558 @@ +//! Associates model tensors with checkpoint sources and prefixed views, +//! over the entries it takes from a checkpoint registry. +//! +//! The store owns those entries, the bindings, and every source and program +//! value (its arena). Views and sources point to it: once `view` was +//! called it must not move, and it must outlive them and every load +//! started from it (`Loader.start` borrows it until `wait` or `cancel`). +//! It borrows the meshes of `Options.meshes`, which belong to the platform. +//! Binding cannot fail: a missing key panics with the key, and what the +//! loader cannot fill is its error (`Loader.start`'s `InvalidBinding`). +const std = @import("std"); + +const stdx = @import("stdx"); + +const Compiler = @import("../Compiler.zig"); +const Exe = @import("../exe.zig").Exe; +const MemorySpace = @import("../platform.zig").Memory.Kind; +const Platform = @import("../platform.zig").Platform; +const Quantization = @import("../quantization.zig").Quantization; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; +const load_log = @import("log.zig").load; +const testing = @import("../testing.zig"); + +const TensorStore = @This(); + +/// Where the loader delivers a loaded tensor, on every device of its mesh +/// (`View.setMemory`). The destination is allocated once per device +/// whatever its pieces; the host kinds take no device memory. +pub const Memory = enum { + /// Device memory. + default, + /// Buffers in each device's pinned host memory space. An executable + /// that takes them as device inputs copies them to the device first. + host_pinned, + /// Pinned host memory that executables read in place: each device's + /// buffer is a device-kind view of it (`createViewOfDeviceBuffer`), + /// and the memory goes with the last view. For a table too large for + /// device memory, of which a step reads a few rows. XLA counts the + /// views against device memory (`View.hostMappedBytes`). + host_mapped, + + /// Whether the bytes live in host memory, written by the CPU. + pub fn isHost(self: Memory) bool { + return switch (self) { + .default => false, + .host_pinned, .host_mapped => true, + }; + } + + /// The memory space of each device's delivered buffer. + pub fn space(self: Memory) MemorySpace { + return switch (self) { + .default, .host_mapped => .default, + .host_pinned => .host_pinned, + }; + } +}; + +/// What the loader reads of a bound tensor (`bindingOf`): the source +/// it fills the tensor from, or the program that computes it. +pub const Binding = union(enum) { + /// The loader fills the tensor from `source` (`Source.Placed.bind`), + /// every byte no piece covers zero. + load: Load, + /// A program computes the tensor while loading (`Source.bindProgram`). + /// Its output lands in default device memory. + compute: Compute, + + pub const Load = struct { + source: Source.Placed, + /// Where the loader delivers this tensor (`View.setMemory`). + memory: Memory = .default, + }; + + pub const Compute = struct { + program: Program, + /// The program's inputs in order, each landed in default device + /// memory, placed by its shape. + inputs: []const Source.Placed, + + /// Whether this computing `output` and `other` computing + /// `other_output` compile to the same executable: equal programs + /// over the same shapes placed alike. + pub fn sameExecutable(self: Compute, output: Shape, other: Compute, other_output: Shape) bool { + if (!self.program.eql(other.program)) return false; + if (!output.eqlWithTagsAndPartitioning(other_output) or self.inputs.len != other.inputs.len) return false; + for (self.inputs, other.inputs) |a, b| { + if (!a.shape.eqlWithTagsAndPartitioning(b.shape)) return false; + } + return true; + } + + /// Compiles the program computing a tensor of `output` from inputs + /// of the shapes it binds, its module named after the program. + pub fn compile( + self: Compute, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + meshes: []const *const Sharding.Mesh, + output: Shape, + ) Compiler.Error!Exe { + const inputs = try allocator.alloc(Tensor, self.inputs.len); + defer allocator.free(inputs); + for (inputs, self.inputs) |*input, source| input.* = .fromShape(source.shape); + const opts: Compiler.Options = .{ .meshes = meshes, .program_name = self.program.name() }; + return self.program.vtable.compile(self.program.value, allocator, io, platform, opts, output, inputs); + } + }; +}; + +/// A program the loader compiles and runs to compute a tensor while loading +/// (`Binding.compute`): `forward(output, inputs)` of the program's value. +/// Model code makes one through `Source.bindProgram`, and reads it back +/// with `as`. +pub const Program = struct { + /// The program's value, in the store's arena. + value: *const anyopaque, + vtable: *const VTable, + + const VTable = struct { + name: []const u8, + compile: *const fn ( + value: *const anyopaque, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Compiler.Options, + output: Shape, + inputs: []const Tensor, + ) Compiler.Error!Exe, + eql: *const fn (a: *const anyopaque, b: *const anyopaque) bool, + }; + + /// `program`, its value copied into `arena`. Its type declares + /// `pub fn forward(self, output: Shape, inputs: []const Tensor) Tensor` + /// and is plain data, without pointers, so equal values are found by + /// value and compile once. + pub fn init(arena: std.mem.Allocator, program: anytype) std.mem.Allocator.Error!Program { + const T = @TypeOf(program); + comptime { + if (!isPlainData(T)) @compileError("TensorStore.Program: " ++ @typeName(T) ++ " must be plain data, without pointers"); + } + const copy = try arena.create(T); + copy.* = program; + return .{ .value = copy, .vtable = &Erased(T).vtable }; + } + + /// The program's value, if it is a `T`. + pub fn as(self: Program, comptime T: type) ?T { + if (self.vtable != &Erased(T).vtable) return null; + return Erased(T).typed(self.value); + } + + /// The program's type name, which names its compiled module in traces, + /// dumps and logs. + pub fn name(self: Program) []const u8 { + return self.vtable.name; + } + + /// Whether `a` and `b` are programs of one type with equal values. + fn eql(a: Program, b: Program) bool { + return a.vtable == b.vtable and a.vtable.eql(a.value, b.value); + } + + /// The vtable of programs of type `T`, one per type. + fn Erased(comptime T: type) type { + return struct { + const vtable: VTable = .{ .name = @typeName(T), .compile = compileValue, .eql = eqlValues }; + + fn typed(erased: *const anyopaque) T { + return @as(*const T, @ptrCast(@alignCast(erased))).*; + } + + fn compileValue( + erased: *const anyopaque, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Compiler.Options, + output: Shape, + inputs: []const Tensor, + ) Compiler.Error!Exe { + return Compiler.compile(allocator, io, platform, T.forward, .{ typed(erased), output, inputs }, opts); + } + + fn eqlValues(a: *const anyopaque, b: *const anyopaque) bool { + return std.meta.eql(typed(a), typed(b)); + } + }; + } + + /// Values compared by value: no pointer anywhere inside. + fn isPlainData(comptime T: type) bool { + return switch (@typeInfo(T)) { + .void, .bool, .int, .float, .@"enum" => true, + .array => |info| isPlainData(info.child), + .optional => |info| isPlainData(info.child), + .@"struct" => |info| inline for (info.field_types) |Field| { + if (!isPlainData(Field)) break false; + } else true, + .@"union" => |info| info.tag_type != null and inline for (info.field_types) |Field| { + if (!isPlainData(Field)) break false; + } else true, + else => false, + }; + } +}; + +pub const Source = @import("Source.zig"); + +registry: safetensors.TensorRegistry, +meshes: []const *const Sharding.Mesh, +quantization: ?Quantization.Target, +id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), +allocator: std.mem.Allocator, +arena: std.heap.ArenaAllocator, + +pub const Options = struct { + /// The meshes `Source.withPartitioning` and `View.mesh` name. The slice + /// is copied; the meshes must outlive the store. + meshes: []const *const Sharding.Mesh = &.{}, + /// How the platform's FP8 GEMMs want block-FP8 weights, for the code + /// that binds them (`View.quantization`): `.of(platform)`. Required to + /// bind block-FP8 weights. + quantization: ?Quantization.Target = null, +}; + +/// A store over the entries of `registry`, which must be complete: add +/// every checkpoint first (`TensorRegistry.addRepo`). It takes the entries +/// and leaves `registry` empty. The store's map stays locked until +/// `deinit`, so a later insert panics in safe builds, and an entry's +/// address is its checkpoint tensor's identity (`View.get`). +pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, opts: Options) TensorStore { + var arena: std.heap.ArenaAllocator = .init(allocator); + const meshes = arena.allocator().dupe(*const Sharding.Mesh, opts.meshes) catch @panic("OOM"); + var self: TensorStore = .{ + .registry = registry.*, + .meshes = meshes, + .quantization = opts.quantization, + .id_to_sources = .empty, + .allocator = allocator, + .arena = arena, + }; + registry.* = .init(registry.arena.child_allocator); + self.registry.tensors.lockPointers(); + return self; +} + +/// Frees the bindings, the sources and programs, then the registry entries +/// `init` took. No load started from the store may still run: `wait` or +/// `cancel` it first. +pub fn deinit(self: *TensorStore) void { + self.id_to_sources.deinit(self.allocator); + self.arena.deinit(); + self.registry.tensors.unlockPointers(); + self.registry.deinit(); +} + +/// The binding of tensor `id`, as the loader reads it; null when this store +/// did not bind it. +pub fn bindingOf(self: *const TensorStore, id: Tensor.Id) ?Binding { + return self.id_to_sources.get(id); +} + +/// The root view, with no prefix. It points to this store, which must not +/// move afterwards. +pub fn view(self: *TensorStore) View { + return .{ .store = self }; +} + +/// The mesh `name` among `Options.meshes`; model code asks its `View`. +pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (store.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + std.debug.panic( + \\Found no meshes named {s} in TensorStore. + \\Try passing more meshes to `zml.io.TensorStore.init`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(store.meshes) }); +} + +/// What model code holds: a key prefix over the store's checkpoint, to look +/// tensors up, bind them and set their memory. A value that keeps its own +/// copy of the prefix (256 bytes at most, longer panics) and points to the +/// store. +pub const View = struct { + store: *TensorStore, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + /// The store's root view, with no prefix. + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + /// This view under `prefix_` and a dot: `withPrefix("model")` then + /// `get("embed")` looks up `model.embed`. It only scopes lookups; + /// nothing is bound. + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix(), prefix_ }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + /// This view under `index` and a dot, as `withPrefix` of the number. + pub fn withLayer(self: *const View, index: usize) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix(), index }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + /// The keys' prefix under this view, with its trailing dot; "" at the + /// root. + pub fn prefix(self: *const View) []const u8 { + return self.prefix_buffer[0..self.prefix_length]; + } + + /// The checkpoint tensor `subkey`, the registry's own entry, whose + /// address is the tensor's identity. Null when the checkpoint has none. + pub fn get(self: View, subkey: []const u8) ?*const safetensors.Tensor { + var buffer: [256]u8 = undefined; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); + return self.store.registry.tensors.getPtr(key); + } + + /// The checkpoint tensor `subkey`, with `tags` (null keeps the + /// checkpoint's), placed on `mesh_` by `partitioning` and bound: + /// `source(subkey, tags).withPartitioning(mesh_, partitioning).bind()`. + /// Panics when the checkpoint has no such tensor. + pub fn createTensor(self: View, subkey: []const u8, comptime tags: anytype, comptime mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); + } + + /// Like `createTensor`, but null when the checkpoint has no such tensor. + pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, comptime mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const source_ = self.maybeSource(subkey, tags) orelse return null; + return source_.withPartitioning(mesh_, partitioning).bind(); + } + + /// `createTensor` replicated on every device, without a mesh. + pub fn createReplicatedTensor(self: View, subkey: []const u8, comptime tags: anytype) Tensor { + return self.createTensor(subkey, tags, .replicated, .{}); + } + + /// Changes where the loader delivers `tensor` (`Memory`), for a + /// decision made after the tensor was bound. `tensor` must be bound in + /// this store, and not computed by a program: a program's output lands + /// in default device memory. + pub fn setMemory(self: View, tensor: Tensor, kind: Memory) void { + const binding = self.store.id_to_sources.getPtr(tensor.id) orelse + stdx.debug.panic("Tensor {} {f} has no binding in this store", .{ tensor.id, tensor.shape() }); + switch (binding.*) { + .load => |*load| load.memory = kind, + .compute => |compute| stdx.debug.panic("setMemory: program {s} computes tensor {} {f}, whose output lands in default device memory", .{ compute.program.name(), tensor.id, tensor.shape() }), + } + } + + /// The bytes on one device of every tensor the store delivers + /// `.host_mapped`, each counted by its shard: what XLA counts against + /// device memory for those views, which a model gives back through + /// `Compiler.Options.extra_device_memory_bytes`. The whole store's, + /// whatever this view's prefix; ask once every binding and `setMemory` + /// is done. A tensor whose mesh partitions do not divide its dimensions + /// is `IncompatibleSharding`, logged. + pub fn hostMappedBytes(self: View) !u64 { + var bytes: u64 = 0; + var it = self.store.id_to_sources.iterator(); + while (it.next()) |entry| switch (entry.value_ptr.*) { + .load => |load| if (load.memory == .host_mapped) { + const shard = Sharding.shardedShape(load.source.shape) catch |err| { + load_log.err("tensor {} {f}, delivered host_mapped: its dimensions are not divisible by its mesh partitions", .{ entry.key_ptr.*, load.source.shape }); + return err; + }; + bytes += shard.byteSize(); + }, + .compute => {}, + }; + return bytes; + } + + /// See `TensorStore.mesh`. + pub fn mesh(self: View, name: @EnumLiteral()) *const Sharding.Mesh { + return self.store.mesh(name); + } + + /// See `Options.quantization`. Panics when the store has none: binding + /// block-FP8 weights requires it. + pub fn quantization(self: View) Quantization.Target { + return self.store.quantization orelse + stdx.debug.panic("TensorStore.Options.quantization is unset: block-FP8 weights bind for a platform; pass .of(platform)", .{}); + } + + /// The checkpoint tensor `subkey` as a `Source`, with `tags` (null keeps + /// the checkpoint's). Panics when the checkpoint has no such tensor. + pub fn source(self: View, subkey: []const u8, comptime tags: anytype) Source { + return self.maybeSource(subkey, tags) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); + } + + /// Like `source`, but null when the checkpoint has no such tensor. + pub fn maybeSource(self: View, subkey: []const u8, comptime tags: anytype) ?Source { + const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); + const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + return .init(self, subkey, if (has_tags) parsed_tags.constSlice() else null); + } + + /// The checkpoint tensors under this view's prefix: a scan of every key. + pub fn count(self: View) usize { + var count_: usize = 0; + const prefix_ = self.prefix(); + var it = self.store.registry.tensors.iterator(); + while (it.next()) |item| { + const key = item.key_ptr.*; + if (std.mem.startsWith(u8, key, prefix_)) { + count_ += 1; + } + } + return count_; + } + + fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { + const key = std.fmt.bufPrint(buffer, fmt, args) catch + std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); + return key; + } +}; + +test "get finds exact keys, the registry's own entries, and count the keys under a prefix" { + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "layer.experts.0.down_proj.weight", .shape = .init(.{4}, .u8), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + // The store took the entries. + try std.testing.expectEqual(0, registry.tensors.count()); + + const layer = store.view().withPrefix("layer"); + const entry = layer.get("experts.0.down_proj.weight").?; + try std.testing.expectEqual(entry, store.registry.tensors.getPtr("layer.experts.0.down_proj.weight").?); + try std.testing.expectEqual(null, layer.get("experts.0.down_proj")); + try std.testing.expectEqual(null, layer.get("weights")); + try std.testing.expectEqualStrings("layer.", layer.prefix()); + try std.testing.expectEqualStrings("", layer.root().prefix()); + try std.testing.expectEqualStrings("layer.experts.0.", layer.withPrefix("experts").withLayer(0).prefix()); + try std.testing.expectEqual(1, layer.withPrefix("experts").count()); + try std.testing.expectEqual(0, layer.withPrefix("gate").count()); + + // Every binding of the entry holds the entry itself. + const tensor = layer.createReplicatedTensor("experts.0.down_proj.weight", null); + try std.testing.expectEqual(entry, store.bindingOf(tensor.id).?.load.source.whole().?); +} + +test "programs of one type and equal values over the same placed shapes share an executable" { + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "t", .shape = .init(.{4}, .u8), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + const Scale = struct { + factor: u8, + + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0]; + } + }; + const Shift = struct { + factor: u8, + + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0]; + } + }; + const shape: Shape = .init(.{4}, .u8); + const input = store.view().source("t", null).withPartitioning(.replicated, .{}); + const tensors = [_]Tensor{ + Source.bindProgram(&.{input}, shape, Scale{ .factor = 1 }), + Source.bindProgram(&.{input}, shape, Scale{ .factor = 1 }), + Source.bindProgram(&.{input}, shape, Scale{ .factor = 2 }), + Source.bindProgram(&.{input}, shape, Shift{ .factor = 1 }), + }; + var computes: [tensors.len]Binding.Compute = undefined; + for (&computes, tensors) |*compute, tensor| compute.* = store.bindingOf(tensor.id).?.compute; + try std.testing.expect(computes[0].sameExecutable(shape, computes[1], shape)); + try std.testing.expect(!computes[0].sameExecutable(shape, computes[2], shape)); + try std.testing.expect(!computes[0].sameExecutable(shape, computes[3], shape)); + try std.testing.expect(!computes[0].sameExecutable(shape, computes[1], shape.withDtype(.i8))); + try std.testing.expectEqual(1, computes[3].program.as(Shift).?.factor); + try std.testing.expect(computes[3].program.as(Scale) == null); +} + +test "setMemory changes where the loader delivers a bound tensor" { + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "table", .shape = .init(.{ 4, 2 }, .u8), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + + const view_ = store.view(); + const tensor = view_.createReplicatedTensor("table", null); + try std.testing.expectEqual(Memory.default, store.bindingOf(tensor.id).?.load.memory); + view_.setMemory(tensor, .host_mapped); + try std.testing.expectEqual(Memory.host_mapped, store.bindingOf(tensor.id).?.load.memory); + view_.setMemory(tensor, .default); + try std.testing.expectEqual(Memory.default, store.bindingOf(tensor.id).?.load.memory); +} + +test "hostMappedBytes counts one device's shard of each host_mapped tensor" { + const allocator = std.testing.allocator; + var devices: testing.Devices = undefined; + try devices.init(allocator, std.testing.io, 2); + defer devices.deinit(); + const tp = try devices.mesh("tp"); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "rows", .shape = .init(.{ 8, 2 }, .u8), .offset = 0 }); + try registry.registerTensor(.{ .file_uri = "weights", .name = "scale", .shape = .init(.{4}, .u8), .offset = 16 }); + try registry.registerTensor(.{ .file_uri = "weights", .name = "dense", .shape = .init(.{4}, .u8), .offset = 20 }); + try registry.registerTensor(.{ .file_uri = "weights", .name = "odd", .shape = .init(.{ 3, 2 }, .u8), .offset = 24 }); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + + const view_ = store.view(); + const rows = view_.createTensor("rows", .{ .row, .col }, .tp, .{ .row = .model }); + const scale = view_.createReplicatedTensor("scale", null); + _ = view_.createReplicatedTensor("dense", null); + try std.testing.expectEqual(0, try view_.hostMappedBytes()); + view_.setMemory(rows, .host_mapped); + view_.setMemory(scale, .host_mapped); + // Half the rows, and the whole replicated scale, from any prefix. + try std.testing.expectEqual(8 + 4, try view_.withPrefix("rows").hostMappedBytes()); + + // Three rows do not split over two devices. + const odd = view_.createTensor("odd", .{ .row, .col }, .tp, .{ .row = .model }); + view_.setMemory(odd, .host_mapped); + try std.testing.expectError(error.IncompatibleSharding, view_.hostMappedBytes()); +} diff --git a/zml/io/backend.zig b/zml/io/backend.zig new file mode 100644 index 00000000..f118f5fe --- /dev/null +++ b/zml/io/backend.zig @@ -0,0 +1,625 @@ +//! The seam between the front end (loader.zig) and its backends: the +//! `LoadSpec` contract they trust, its parts' byte runs (`Lattice`), the +//! source request size (`requestSize`), and which backend and memory kinds +//! a platform takes (`loadsDirect`, `supportsMemory`). +const std = @import("std"); + +const VFS = @import("vfs"); + +const Buffer = @import("../buffer.zig").Buffer; +const Platform = @import("../platform.zig").Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Memory = @import("TensorStore.zig").Memory; +const constants = @import("../constants.zig"); +const buffered_loader = @import("buffered_loader.zig"); +const direct_loader = @import("direct_loader.zig"); +const dma_calibration = @import("dma_calibration.zig"); + +/// The largest source request, and so the largest DMA block: a request is +/// a whole number of blocks, at least one. +pub const max_read_request_size: usize = 32 * 1024 * 1024; + +/// The source request for DMA blocks of `block_size`: the read chunk +/// rounded up to whole blocks, one block at least. Null unless the block is +/// a nonzero multiple of `alignment` (`readAlignment`: reads land in +/// blocks), and the request is at most `max_read_request_size` and, with an +/// alignment, longer than two units, so a read widened by a unit at each +/// end fits. `zml.io.Loader.init` refuses options for which a block +/// calibration may select has none. +pub fn requestSize(read_chunk_size: usize, block_size: usize, alignment: usize) ?usize { + if (block_size == 0 or block_size % @max(alignment, 1) != 0) return null; + // Bounded before the rounding, which then cannot overflow. + const least = @max(read_chunk_size, block_size); + if (least > max_read_request_size) return null; + const request = std.mem.alignForwardAnyAlign(usize, least, block_size); + if (request > max_read_request_size) return null; + if (alignment != 0 and request <= 2 * alignment) return null; + return request; +} + +/// What the direct backend widens the reads of a file read directly to: +/// the VFS's alignment when `io` is a VFS and `direct_io` is not `off`, else +/// 0, every read exact. The VFS decides per local file whether it is read +/// directly (`VFS.useDirectIo`). +pub fn readAlignment(io: std.Io, direct_io: VFS.DirectIo) usize { + if (direct_io == .off or VFS.fromIo(io) == null) return 0; + return VFS.direct_io_alignment; +} + +/// `zml.io.Loader.Options` as `zml.io.Loader.init` checked them, the +/// checkpoint resolved to its VFS backend's read hints and the read width to +/// its default. See `zml.io.Loader.Options` for each field. +pub const Options = struct { + read_parallelism: usize, + read_hints: VFS.ReadHints = .local, + dma_block_size: ?usize = null, + direct_io: VFS.DirectIo = .auto, + /// The loader's checkpoint (loader.zig, `Loader.Options.checkpoint`) + /// when it is local and the VFS may read it direct (`readAlignment` is + /// not 0), else null; owned by the loader's set-up task. The direct + /// backend predicts from it how the checkpoint's files read, places its + /// arenas for that, and reads each of those files as predicted + /// (`direct_loader.Loader.create`); it keeps nothing of the path. + checkpoint: ?[]const u8 = null, + + /// The blocks calibration may select: `dma_block_size`, else + /// `dma_calibration.default_block_sizes`. + pub fn dmaBlockSizes(self: *const Options) []const usize { + return if (self.dma_block_size) |*block_size| block_size[0..1] else &dma_calibration.default_block_sizes; + } +}; + +/// One destination, its caller-owned output and the source ranges that fill +/// it, as the front end checked them (`checkParts`, `Backend.stridedRead`): +/// backends trust them. +/// Everything a spec names (its parts, their dimensions, the sources, the +/// output) stays alive until its await; backends borrow it. +pub const LoadSpec = struct { + /// The destination's global shape, which the output buffer keeps. Its + /// sharding places the bytes on the devices; a shape without a mesh is + /// replicated over every device (`Platform.placedShape`). + shape: Shape, + /// Where the bytes go, from the source's `TensorStore` binding. The + /// front end only submits kinds `supportsMemory` accepts, and a host + /// kind only with contiguous parts (`Lattice.dims` empty). + memory: Memory = .default, + /// Written whole once the destination's buffers exist, and not touched + /// before: a failure leaves it as it was or holding those buffers. + output: *Buffer, + /// What fills the destination's logical bytes, in ascending, disjoint + /// destination order; every byte no part covers is zero. + parts: []const LoadPart, +}; + +/// The bytes of `source` that `lattice` moves: its source offsets relative +/// to the tensor's data, its destination offsets into the destination's +/// logical bytes, the row-major bytes of its packed shape before the +/// sharding splits them. +pub const LoadPart = struct { + source: *const safetensors.Tensor, + lattice: Lattice, + + /// The whole of `source`, at the start of the destination. + pub fn whole(source: *const safetensors.Tensor) LoadPart { + return .{ .source = source, .lattice = .{ .width = source.byteSize() } }; + } +}; + +/// The most dimensions a part repeats over: `Source` gives each axis at +/// most two. +pub const max_part_dims = 2 * constants.MAX_RANK; + +/// A part's byte moves: the run of `width` bytes at `src_offset` and +/// `dst_offset`, and, for a strided part, one more for every index of +/// `dims` (outermost first), each offset moved by the index times its +/// strides. A zero source stride repeats the same bytes. Runs are numbered +/// row-major over `dims`. +pub const Lattice = struct { + src_offset: u64 = 0, + dst_offset: u64 = 0, + width: u64, + /// Empty for one contiguous run. + dims: []const Dim = &.{}, + + /// A step of the runs: `count` of them, each moving the source and + /// destination offsets by their strides. + pub const Dim = struct { + count: u64, + src_stride: u64, + dst_stride: u64, + }; + + pub const Run = struct { src: u64, dst: u64 }; + + pub fn runs(self: Lattice) u64 { + var n: u64 = 1; + for (self.dims) |d| n *= d.count; + return n; + } + + pub fn at(self: Lattice, index: u64) Run { + var rest = index; + var run: Run = .{ .src = self.src_offset, .dst = self.dst_offset }; + var d = self.dims.len; + while (d > 0) { + d -= 1; + const i = rest % self.dims[d].count; + rest /= self.dims[d].count; + run.src += i * self.dims[d].src_stride; + run.dst += i * self.dims[d].dst_stride; + } + return run; + } + + /// The source and destination bytes the runs span from their first + /// offsets, null when a count is zero or the bounds overflow. + pub fn spans(self: Lattice) ?struct { src: u64, dst: u64 } { + var src = self.width; + var dst = self.width; + for (self.dims) |d| { + if (d.count == 0) return null; + src = std.math.add(u64, std.math.mul(u64, d.count - 1, d.src_stride) catch return null, src) catch return null; + dst = std.math.add(u64, std.math.mul(u64, d.count - 1, d.dst_stride) catch return null, dst) catch return null; + } + return .{ .src = src, .dst = dst }; + } + + /// Whether each dimension steps past everything inside it on the + /// destination side: runs land in index order without overlapping. + /// Needs `spans` not null. + pub fn dstOrdered(self: Lattice) bool { + return ordered(self.dims, self.width, .dst, false); + } + + /// Whether the source runs come in index order without overlapping, + /// gaps allowed: the runs of an index range lie between the first's + /// start and the last's end. Needs `spans` not null. + pub fn srcOrdered(self: Lattice) bool { + return ordered(self.dims, self.width, .src, false); + } + + /// Whether the source runs are back to back in index order: an index + /// range reads one contiguous source range, each byte once. Needs + /// `spans` not null. + pub fn srcTiled(self: Lattice) bool { + return ordered(self.dims, self.width, .src, true); + } + + /// Whether each dimension steps past everything inside it on `side`, + /// exactly with `exact`. The extents it sums are bounded by `spans`. + fn ordered(dims: []const Dim, width: u64, comptime side: enum { src, dst }, exact: bool) bool { + var extent = width; + var d = dims.len; + while (d > 0) { + d -= 1; + if (dims[d].count < 2) continue; + const stride = if (side == .src) dims[d].src_stride else dims[d].dst_stride; + if (if (exact) stride != extent else stride < extent) return false; + extent += (dims[d].count - 1) * stride; + } + return true; + } + + /// The first run, in index order, that ends after destination offset + /// `offset`; `runs()` when none does. Needs `dstOrdered`. + pub fn firstEndingAfter(self: Lattice, offset: u64) u64 { + var low: u64 = 0; + var high = self.runs(); + while (low < high) { + const middle = low + (high - low) / 2; + if (self.at(middle).dst + self.width > offset) high = middle else low = middle + 1; + } + return low; + } + + /// The first run that starts at destination offset `offset` or later; + /// `runs()` when none does. Needs `dstOrdered`. + pub fn firstStartingFrom(self: Lattice, offset: u64) u64 { + var low: u64 = 0; + var high = self.runs(); + while (low < high) { + const middle = low + (high - low) / 2; + if (self.at(middle).dst >= offset) high = middle else low = middle + 1; + } + return low; + } + + /// The destination bytes of the runs inside `[start, end)`. + pub fn coveredIn(self: Lattice, start: u64, end: u64) u64 { + const first = self.firstEndingAfter(start); + const last = self.firstStartingFrom(end); + if (first >= last) return 0; + var covered = (last - first) * self.width; + const head = self.at(first).dst; + if (head < start) covered -= start - head; + const tail = self.at(last - 1).dst + self.width; + if (tail > end) covered -= tail - end; + return covered; + } + + /// Runs `first` to `end` (exclusive), in index order, without a + /// division per run. + pub const Cursor = struct { + lattice: Lattice, + index: u64, + end: u64, + counters: [max_part_dims]u64, + run: Run, + + pub fn init(lattice: Lattice, first: u64, end: u64) Cursor { + std.debug.assert(lattice.dims.len <= max_part_dims); + var cursor: Cursor = .{ .lattice = lattice, .index = first, .end = end, .counters = undefined, .run = lattice.at(first) }; + var rest = first; + var d = lattice.dims.len; + while (d > 0) { + d -= 1; + cursor.counters[d] = rest % lattice.dims[d].count; + rest /= lattice.dims[d].count; + } + return cursor; + } + + pub fn next(self: *Cursor) ?Run { + if (self.index >= self.end) return null; + const current = self.run; + self.index += 1; + if (self.index < self.end) { + const dims = self.lattice.dims; + var d = dims.len; + while (d > 0) { + d -= 1; + self.counters[d] += 1; + self.run.src +%= dims[d].src_stride; + self.run.dst +%= dims[d].dst_stride; + if (self.counters[d] < dims[d].count) break; + self.counters[d] = 0; + self.run.src -%= dims[d].count * dims[d].src_stride; + self.run.dst -%= dims[d].count * dims[d].dst_stride; + } + } + return current; + } + }; + + /// Copies runs `first` to `end` (exclusive): the `width` bytes at + /// `src + run.src` to `dst + run.dst`, `src` and `dst` being the + /// addresses the offsets count from (wrapping). Whole rows of the two + /// innermost dimensions go through nested loops, and a run of 1, 2, 4, + /// 8 or 16 bytes is one load and one store: a swizzle's 4-byte runs + /// cost a few cycles each instead of a cursor step and a `memcpy`. + pub fn copyRuns(self: Lattice, first: u64, end: u64, dst: usize, src: usize) void { + switch (self.width) { + inline 1, 2, 4, 8, 16 => |width| self.copyRunsOf(width, first, end, dst, src), + else => self.copyRunsOf(0, first, end, dst, src), + } + } + + /// `copyRuns` for runs of `width` bytes, or of `self.width` when zero. + fn copyRunsOf(self: Lattice, comptime width: usize, first: u64, end: u64, dst: usize, src: usize) void { + if (first >= end) return; + const inner = self.dims[self.dims.len - @min(self.dims.len, 2) ..]; + var row: u64 = 1; + for (inner) |d| row *= d.count; + const rows_start = std.math.divCeil(u64, first, row) catch unreachable; + const rows_end = end / row; + if (rows_start >= rows_end) return self.copyEach(width, first, end, dst, src); + self.copyEach(width, first, rows_start * row, dst, src); + const outer: Lattice = .{ .src_offset = self.src_offset, .dst_offset = self.dst_offset, .width = self.width, .dims = self.dims[0 .. self.dims.len - inner.len] }; + var rows: Cursor = .init(outer, rows_start, rows_end); + while (rows.next()) |start| { + const row_dst = dst +% start.dst; + const row_src = src +% start.src; + switch (inner.len) { + 0 => copyRun(width, self.width, row_dst, row_src), + 1 => copyLine(width, self.width, inner[0], row_dst, row_src), + else => { + var line_dst = row_dst; + var line_src = row_src; + for (0..inner[0].count) |_| { + copyLine(width, self.width, inner[1], line_dst, line_src); + line_dst +%= inner[0].dst_stride; + line_src +%= inner[0].src_stride; + } + }, + } + } + self.copyEach(width, rows_end * row, end, dst, src); + } + + fn copyEach(self: Lattice, comptime width: usize, first: u64, end: u64, dst: usize, src: usize) void { + if (first >= end) return; + var cursor: Cursor = .init(self, first, end); + while (cursor.next()) |run| copyRun(width, self.width, dst +% run.dst, src +% run.src); + } + + inline fn copyLine(comptime width: usize, len: u64, d: Dim, dst: usize, src: usize) void { + var to = dst; + var from = src; + for (0..d.count) |_| { + copyRun(width, len, to, from); + to +%= d.dst_stride; + from +%= d.src_stride; + } + } + + inline fn copyRun(comptime width: usize, len: u64, dst: usize, src: usize) void { + if (width == 0) { + const n: usize = @intCast(len); + @memcpy(@as([*]u8, @ptrFromInt(dst))[0..n], @as([*]const u8, @ptrFromInt(src))[0..n]); + } else { + const Word = @Int(.unsigned, 8 * width); + @as(*align(1) Word, @ptrFromInt(dst)).* = @as(*align(1) const Word, @ptrFromInt(src)).*; + } + } +}; + +/// Whether `parts` keep the `LoadSpec` contract for a destination of +/// `destination_bytes`: there is at least one, each is non-empty and inside +/// its source and the destination, and they come in ascending destination +/// order, each starting at or after the previous contiguous part's end or +/// strided part's start. Each part's bytes end in their file at least +/// `VFS.direct_io_alignment` below `maxInt(u64)`, so a read of them widened +/// to the alignment does not overflow. Strided parts must land in index +/// order (`Lattice.dstOrdered`); interleaving ones are only checked against +/// each other by the bytes they cover together, since only `Source` makes +/// them, disjoint by construction. +pub fn checkParts(parts: []const LoadPart, destination_bytes: u64) bool { + if (parts.len == 0) return false; + var floor: u64 = 0; + var covered: u64 = 0; + for (parts) |part| { + const lattice = part.lattice; + if (lattice.width == 0 or lattice.dims.len > max_part_dims) return false; + const spans = lattice.spans() orelse return false; + if (!lattice.dstOrdered()) return false; + const source_end = std.math.add(u64, lattice.src_offset, spans.src) catch return false; + const file_end = std.math.add(u64, part.source.offset, source_end) catch return false; + const end = std.math.add(u64, lattice.dst_offset, spans.dst) catch return false; + if (source_end > part.source.byteSize() or file_end > std.math.maxInt(u64) - VFS.direct_io_alignment) return false; + if (end > destination_bytes or lattice.dst_offset < floor) return false; + floor = if (lattice.dims.len == 0) end else lattice.dst_offset; + covered +|= lattice.runs() * lattice.width; + } + return covered <= destination_bytes; +} + +/// Whether `platform` loads through the direct backend. Transfer-manager +/// support and host pinning are independent. CPU implements byte-range +/// transfers from ordinary pages but not DmaMap, so it shares coalescing +/// and bounded blocks without DMA support. The whole-tensor buffered path +/// has neither: on one B70, interleaved HF Qwen3.5-4B loads took 10.9 to +/// 16.3 s direct against about 48 s buffered. Furiosa stages in its +/// plugin's `.host_pinned` buffers (SDK dma-buf memory, which the SDK +/// transfers without pinning pages per request). TPU, neuron and metal +/// stay buffered until their transfer path is checked on hardware. +pub fn loadsDirect(platform: *const Platform) bool { + return switch (platform.target) { + .cuda, .rocm, .oneapi, .cpu, .furiosa => true, + .tpu, .neuron, .metal => false, + }; +} + +/// Whether the loader can deliver into `kind` on `platform`. The host +/// kinds need the direct pipeline and a pinned host memory space on every +/// device. CPU loads through the direct backend, so it accepts them (its +/// plugin resolves every kind to its one memory space). +pub fn supportsMemory(platform: *const Platform, kind: Memory) bool { + if (kind == .default) return true; + if (!loadsDirect(platform)) return false; + for (platform.devices) |*device| { + if (device.memory(.host_pinned) == null) return false; + } + return true; +} + +/// Bytes per device, by `platform.devices` index: `Platform.init` refuses +/// more devices than it holds. +pub const DeviceBytes = [Platform.MAX_NUM_DEVICES]u64; + +/// The transfer implementation selected for the platform (`loadsDirect`). +pub const Backend = union(enum) { + direct: *direct_loader.Loader, + buffered: *buffered_loader.Loader, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Options, + ) !Backend { + return if (loadsDirect(platform)) + .{ .direct = try direct_loader.Loader.create(allocator, io, platform, opts) } + else + .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, opts) }; + } + + /// Publishes `specs`, and completes one item of `progress` per spec. + /// Fails with what the backend detected in the files, host memory and + /// PJRT, logged there (a plain allocation failure is `OutOfMemory`, + /// unlogged). + pub fn submit(self: Backend, specs: []const LoadSpec, progress: std.Progress.Node) !Submission { + return switch (self) { + inline else => |loader, tag| @unionInit(Submission, @tagName(tag), try loader.submit(specs, progress)), + }; + } + + /// `read`, the source bytes every job of `part` reads whole, widened to + /// its file's alignment (0 when its jobs read only their own range: + /// source runs back to back, a contiguous part included), and `limit`, + /// the most one read takes: the front end refuses a part whose `read` + /// passes `limit`. Needs `checkParts`. The direct backend's limit is its + /// smallest read, whichever block calibration picks: the checkpoint's + /// read chunk (8 MiB locally) rounded up to whole blocks of the smallest + /// it may pick, 2 MiB or a pinned `dma_block_size` + /// (`direct_loader.Loader.stridedRead`); the buffered backend reads any + /// part. + pub fn stridedRead(self: Backend, part: LoadPart) StridedRead { + return switch (self) { + .direct => |loader| loader.stridedRead(part), + .buffered => .{ .read = 0, .limit = std.math.maxInt(u64) }, + }; + } + + pub const StridedRead = struct { read: u64, limit: u64 }; + + /// Waits for and retires `submission`, which this backend published. + /// Its batch is dangling afterwards. + pub fn await(self: Backend, submission: Submission) !void { + return switch (self) { + inline else => |loader, tag| loader.awaitBatch(@field(submission, @tagName(tag))), + }; + } + + /// Fails the backend unless it already failed: queued work is dropped, + /// work in flight drains, and every later await returns the first error. + pub fn abort(self: Backend) void { + switch (self) { + inline else => |loader| loader.abort(), + } + } + + pub fn destroy(self: Backend) void { + switch (self) { + inline else => |loader| loader.destroy(), + } + } +}; + +/// A published batch of the backend that published it. +pub const Submission = union(enum) { + direct: *direct_loader.Batch, + buffered: *buffered_loader.Batch, +}; + +/// A random lattice whose runs land in order (`Lattice.dstOrdered`), its +/// source strides anything, zero (a broadcast) included. +fn randomLattice(random: std.Random, dims: []Lattice.Dim) Lattice { + const width = random.intRangeAtMost(u64, 1, 17); + var extent = width; + var d = dims.len; + while (d > 0) { + d -= 1; + dims[d] = .{ + .count = random.intRangeAtMost(u64, 1, 5), + .src_stride = random.intRangeAtMost(u64, 0, 40), + .dst_stride = extent + random.intRangeAtMost(u64, 0, 3) * random.intRangeAtMost(u64, 0, 4), + }; + extent = dims[d].dst_stride * (dims[d].count - 1) + extent; + } + return .{ .src_offset = random.intRangeAtMost(u64, 0, 9), .dst_offset = random.intRangeAtMost(u64, 0, 9), .width = width, .dims = dims }; +} + +test "the cursor walks a range's runs where at places them, and copyRuns copies them" { + var prng: std.Random.DefaultPrng = .init(0x5eed); + const random = prng.random(); + var source: [4096]u8 = undefined; + for (&source, 0..) |*byte, i| byte.* = @truncate(i * 7 + 1); + var expected: [4096]u8 = undefined; + var actual: [4096]u8 = undefined; + var dims_buffer: [5]Lattice.Dim = undefined; + for (0..2000) |_| { + var lattice = randomLattice(random, dims_buffer[0..random.intRangeAtMost(usize, 0, dims_buffer.len)]); + // The widths `copyRuns` moves as one word, and others; a wider run + // may overlap its neighbours, which no part does. + if (random.boolean()) lattice.width = ([_]u64{ 1, 2, 4, 8, 16 })[random.uintLessThan(usize, 5)]; + const spans = lattice.spans().?; + if (lattice.src_offset + spans.src > source.len or lattice.dst_offset + spans.dst > expected.len or !lattice.dstOrdered()) continue; + const runs = lattice.runs(); + const first = random.uintAtMost(u64, runs); + const end = random.intRangeAtMost(u64, first, runs); + @memset(&expected, 0); + @memset(&actual, 0); + var cursor: Lattice.Cursor = .init(lattice, first, end); + for (first..end) |index| { + const run = lattice.at(index); + try std.testing.expectEqual(run, cursor.next().?); + const width: usize = @intCast(lattice.width); + @memcpy(expected[@intCast(run.dst)..][0..width], source[@intCast(run.src)..][0..width]); + } + try std.testing.expectEqual(null, cursor.next()); + lattice.copyRuns(first, end, @intFromPtr(&actual), @intFromPtr(&source)); + try std.testing.expectEqualSlices(u8, &expected, &actual); + } +} + +test "checkParts refuses parts outside their source or destination, out of order or overlapping" { + // A source of 16 bytes into a destination of 32. + const source: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = 0 }; + // Sixteen bytes ending at the largest file offset a read widened to the + // alignment can reach, and one byte later. + const last: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = std.math.maxInt(u64) - VFS.direct_io_alignment - 16 }; + const past_last: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = last.offset + 1 }; + // Runs of 4 bytes, every other 4 bytes of the destination. + const every_other: []const Lattice.Dim = &.{.{ .count = 4, .src_stride = 4, .dst_stride = 8 }}; + const refused = [_][]const LoadPart{ + &.{}, + &.{.{ .source = &source, .lattice = .{ .width = 0 } }}, + // Past the source, past the destination, an overflow, and past the + // largest file offset. + &.{.{ .source = &source, .lattice = .{ .src_offset = 8, .width = 9 } }}, + &.{.{ .source = &source, .lattice = .{ .dst_offset = 24, .width = 9 } }}, + &.{.{ .source = &source, .lattice = .{ .src_offset = std.math.maxInt(u64), .width = 1 } }}, + &.{.whole(&past_last)}, + // Overlapping, and descending. + &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 8 } } }, + &.{ .{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4 } }, .{ .source = &source, .lattice = .{ .width = 4 } } }, + // Strided: runs that overlap on the destination, a dimension of no + // step, strides whose span overflows, past the source, past the + // destination, and descending. + &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 2 }} } }}, + &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 0, .src_stride = 4, .dst_stride = 8 }} } }}, + &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 1 << 40, .src_stride = 0, .dst_stride = 1 << 40 }} } }}, + &.{.{ .source = &source, .lattice = .{ .src_offset = 4, .width = 4, .dims = every_other } }}, + &.{.{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4, .dims = every_other } }}, + &.{ .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } } }, + // Interleaving strided parts that cover more than the destination, + // and contiguous parts overlapping beside a strided one. + &.{ .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } } }, + &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 16, .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 8 }} } } }, + }; + for (refused) |parts| try std.testing.expect(!checkParts(parts, 32)); + const accepted = [_][]const LoadPart{ + // Back to back. + &.{ .{ .source = &source, .lattice = .{ .width = 16 } }, .{ .source = &source, .lattice = .{ .dst_offset = 16, .width = 16 } } }, + // Two strided parts interleaving their runs. + &.{ .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } } }, + // A contiguous part, then a strided one. + &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 8 }} } } }, + // A broadcast: one source run repeated over the destination. + &.{.{ .source = &source, .lattice = .{ .width = 8, .dims = &.{.{ .count = 4, .src_stride = 0, .dst_stride = 8 }} } }}, + // Ending at the largest file offset. + &.{.whole(&last)}, + }; + for (accepted) |parts| try std.testing.expect(checkParts(parts, 32)); +} + +test "requestSize rounds the read chunk up to whole blocks of whole alignment units" { + const mib: usize = 1 << 20; + // One block at least, the chunk rounded up to whole blocks, and a + // rounded request past the largest refused. + try std.testing.expectEqual(16 * mib, requestSize(8 * mib, 16 * mib, 4096)); + try std.testing.expectEqual(9 * mib, requestSize(8 * mib, 3 * mib, 4096)); + try std.testing.expectEqual(null, requestSize(32 * mib, 24 * mib, 0)); + // Without an alignment any nonzero block; with one, only its multiples. + try std.testing.expectEqual(8 * mib + 2048, requestSize(8 * mib, 8 * mib + 2048, 0)); + try std.testing.expectEqual(null, requestSize(8 * mib, 8 * mib + 2048, 4096)); + try std.testing.expectEqual(null, requestSize(8 * mib, 0, 0)); + // A request of two alignment units cannot hold a read widened at both + // ends. + try std.testing.expectEqual(null, requestSize(4096, 8192, 4096)); + try std.testing.expectEqual(3 * 4096, requestSize(4096, 3 * 4096, 4096)); +} + +test "the options pin a DMA block or leave calibration its candidates, and only a VFS widens reads" { + const pinned: Options = .{ .read_parallelism = 1, .dma_block_size = 4 << 20 }; + try std.testing.expectEqualSlices(usize, &.{4 << 20}, pinned.dmaBlockSizes()); + const calibrated: Options = .{ .read_parallelism = 1 }; + try std.testing.expectEqualSlices(usize, &dma_calibration.default_block_sizes, calibrated.dmaBlockSizes()); + + var filesystem = try VFS.init(std.testing.allocator, std.testing.io); + defer filesystem.deinit(); + try std.testing.expectEqual(VFS.direct_io_alignment, readAlignment(filesystem.io(), .auto)); + try std.testing.expectEqual(0, readAlignment(filesystem.io(), .off)); + try std.testing.expectEqual(0, readAlignment(std.testing.io, .on)); +} diff --git a/zml/io/buffered_loader.zig b/zml/io/buffered_loader.zig new file mode 100644 index 00000000..96cfac3e --- /dev/null +++ b/zml/io/buffered_loader.zig @@ -0,0 +1,839 @@ +//! Whole-tensor staging for platforms that use Buffer.from: TPU, neuron +//! and metal, until the direct path is checked on them. A composite +//! destination is assembled in the staging: its gaps zeroed, each +//! part read at its offset (a strided part reads its source range once and +//! copies each run where it lands), then the whole tensor is placed as any +//! other. Each batch runs up to the read width of workers over its specs; +//! read permits bound the source reads and one byte budget the host +//! staging, live and kept for reuse. There is no calibration and no pinned +//! memory. The shared front end owns source lookup, programs and admission. +const std = @import("std"); + +const stdx = @import("stdx"); +const VFS = @import("vfs"); + +const Buffer = @import("../buffer.zig").Buffer; +const Platform = @import("../platform.zig").Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const backend = @import("backend.zig"); +const testing = @import("../testing.zig"); +const LoadPart = backend.LoadPart; +const LoadSpec = backend.LoadSpec; +const log = @import("log.zig").load; + +/// The host staging budget in largest requests: this many times the +/// largest staging request submitted (`stagingBytes`), for the buffers in +/// use and those kept for reuse alike. Twelve rather than the read width: +/// at a high-latency width of 32 the read width would nearly triple the +/// host staging for nothing this path can use (on such a source a tensor +/// splits its read across the free permits). +const staging_tensors: usize = 12; + +pub const Loader = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + /// Workers per batch, so the threads grow with the batches in flight: + /// the read permits bound their reads and the staging budget their + /// bytes, not their number. + width: usize, + /// The smallest claim of a tensor's read split across helpers + /// (`ChunkedRead.claim`). A tensor is one positional read unless spare + /// permits split it on a high-latency source. + read_chunk_size: usize, + /// The widest fan-out one tensor may take. One on a source whose reads + /// are bandwidth-bound rather than round-trip bound: splitting there only + /// spends tasks, and measurably so -- a replicated local load, whose long + /// transfers keep permits free, loses time to helpers it has no use for. + tensor_workers: usize, + permits: ReadPermits, + staging: Staging, + first_error: stdx.FirstError(anyerror) = .{}, + + pub fn create( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: backend.Options, + ) !*Loader { + const self = try allocator.create(Loader); + self.* = .{ + .allocator = allocator, + .io = io, + .platform = platform, + .width = opts.read_parallelism, + .staging = .{ .slots = @min(opts.read_parallelism, staging_tensors) }, + .read_chunk_size = opts.read_hints.read_chunk_size, + // A high-latency source may use spare read permits to split a + // tensor already staging: more concurrency without more host memory. + .tensor_workers = if (opts.read_hints.high_latency) opts.read_parallelism else 1, + .permits = .{ .free = opts.read_parallelism }, + }; + return self; + } + + /// Starts up to `width` workers over the specs, each assembling one + /// destination at a time from its parts. The batch borrows `specs` and + /// everything they name until it is awaited (`backend.LoadSpec`). On an + /// io that starts no worker, the batch loads inside this call. Nothing + /// runs when this fails. + pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { + try self.checkOpen(); + var largest: usize = 0; + for (specs) |spec| largest = @max(largest, stagingBytes(spec)); + const batch = try self.allocator.create(Batch); + batch.* = .{ .specs = specs, .progress = progress }; + self.staging.widen(self.io, largest); + // `concurrent`, never `async`: a busy Threaded io runs `async` + // inline, and an inline first worker would drain the whole cursor + // here. + var started: usize = 0; + for (0..@min(self.width, specs.len)) |_| { + batch.workers.concurrent(self.io, work, .{ self, batch }) catch break; + started += 1; + } + if (started == 0) self.work(batch); + return batch; + } + + /// Waits for the batch's workers, frees it and returns the sticky error. + /// Uncancelable: the workers borrow the batch and its specs until they + /// return. + pub fn awaitBatch(self: *Loader, batch: *Batch) !void { + const protection = self.io.swapCancelProtection(.blocked); + batch.workers.await(self.io) catch |err| switch (err) { + error.Canceled => unreachable, + }; + _ = self.io.swapCancelProtection(protection); + self.allocator.destroy(batch); + try self.checkOpen(); + } + + /// Fails the loader unless it already failed: workers start no other + /// tensor, and one waiting for its staging returns once it gets it. + pub fn abort(self: *Loader) void { + self.recordError(error.Canceled, "loader aborted", .{}); + } + + /// Every batch was awaited, so no worker runs. + pub fn destroy(self: *Loader) void { + self.staging.deinit(self.allocator); + self.allocator.destroy(self); + } + + fn recordError(self: *Loader, err: anyerror, comptime fmt: []const u8, args: anytype) void { + if (self.first_error.set(err)) log.failure(err, fmt, args); + } + + fn checkOpen(self: *Loader) !void { + if (self.first_error.get()) |err| return err; + } + + /// One of the batch's workers: loads the next unclaimed spec until none + /// is left. + fn work(self: *Loader, batch: *Batch) void { + while (true) { + const index = batch.next.fetchAdd(1, .monotonic); + if (index >= batch.specs.len) return; + self.loadOne(batch.specs[index]); + // One item on the caller's node: a child node per tensor runs + // out of std.Progress's 127. + batch.progress.completeOne(); + } + } + + fn loadOne(self: *Loader, spec: LoadSpec) void { + if (self.first_error.get() != null) return; + std.debug.assert(spec.parts.len != 0); + const tensor_bytes = spec.shape.byteSize(); + const staging = self.staging.acquire(self.allocator, self.io, stagingBytes(spec)) catch |err| + return self.fail(err, "allocate staging for", spec, &spec.parts[0]); + defer self.staging.release(self.allocator, self.io, staging); + // The staging may have taken a while: a failure meanwhile leaves + // this tensor unstarted. + if (self.first_error.get() != null) return; + const tensor = staging[0..tensor_bytes]; + // Staging is reused, so a byte no part writes would publish another + // tensor's bytes rather than zero: the gaps are zeroed first. + var strided = false; + for (spec.parts) |part| strided = strided or part.lattice.dims.len != 0; + if (strided) { + @memset(tensor, 0); + } else { + // Only the gaps: the parts overwrite the rest. + var covered_end: usize = 0; + for (spec.parts) |part| { + const start: usize = @intCast(part.lattice.dst_offset); + @memset(tensor[covered_end..start], 0); + covered_end = start + @as(usize, @intCast(part.lattice.width)); + } + @memset(tensor[covered_end..], 0); + } + for (spec.parts) |*part| { + const file = std.Io.Dir.openFile(.cwd(), self.io, part.source.file_uri, .{ .mode = .read_only }) catch |err| + return self.fail(err, "open source for", spec, part); + defer file.close(self.io); + const lattice = part.lattice; + if (lattice.dims.len == 0) { + const start: usize = @intCast(lattice.dst_offset); + self.readInto(part.source, file, lattice.src_offset, tensor[start..][0..@intCast(lattice.width)]) catch |err| + return self.fail(err, "read", spec, part); + continue; + } + // A strided part: its source range once, into the staging past + // the tensor, then each run where it lands. + const source = staging[tensor_bytes..][0..@intCast(lattice.spans().?.src)]; + self.readInto(part.source, file, lattice.src_offset, source) catch |err| + return self.fail(err, "read", spec, part); + lattice.copyRuns(0, lattice.runs(), @intFromPtr(tensor.ptr), @intFromPtr(source.ptr) -% @as(usize, @intCast(lattice.src_offset))); + } + // The front end asks no host kind of this backend + // (`backend.supportsMemory`). + std.debug.assert(spec.memory == .default); + // The shape's sharding places the bytes; a shape without a mesh is + // replicated, as `Buffer.from` places it. + spec.output.* = Buffer.from(self.io, self.platform, spec.shape, tensor, .{ .wait = true }) catch |err| + return self.fail(err, "transfer", spec, &spec.parts[0]); + } + + /// Fails the loader with `err`, which `step` of `spec` met at `part`. + fn fail(self: *Loader, err: anyerror, step: []const u8, spec: LoadSpec, part: *const LoadPart) void { + self.recordError(err, "{s} tensor {s}: file={s}, offset={d}, source_bytes={d}, parts={d}, destination_bytes={d}", .{ step, part.source.name, part.source.file_uri, part.source.offset, part.source.byteSize(), spec.parts.len, spec.shape.byteSize() }); + } + + /// Fills `destination` from `source`'s bytes at `source_offset`, read + /// from `file`, splitting it across helper tasks when the source permits + /// are free. The permits are released before the caller's transfer, so a + /// tensor staging to the device lends its read budget to one still + /// reading. + fn readInto( + self: *Loader, + source: *const safetensors.Tensor, + file: std.Io.File, + source_offset: u64, + destination: []u8, + ) safetensors.Tensor.RangeError!void { + var chunked: ChunkedRead = .{ + .io = self.io, + .source = source, + .file = file, + .source_offset = source_offset, + .destination = destination, + .chunk_size = self.read_chunk_size, + }; + // Splitting below the read chunk buys nothing, so that is the + // most workers this tensor can use. + var useful_workers = @min( + self.tensor_workers, + destination.len / self.read_chunk_size + + @intFromBool(destination.len % self.read_chunk_size != 0), + ); + + self.permits.acquire(self.io); + var held: usize = 1; + var group: std.Io.Group = .init; + while (true) { + // Helpers only ever take permits that are already free, so a busy + // source sees one read per tensor. The check repeats between + // claims because the tensors that finish first hand their budget + // to the one still reading -- which is the long tensor that + // needed the help. + // Nothing unclaimed means a helper would only start and stop. + if (held < useful_workers and chunked.next.load(.monotonic) < destination.len) { + const extra = self.permits.tryAcquire(self.io, useful_workers - held); + if (extra != 0) { + // Claims size themselves by the width, so it is raised + // before a helper can claim. `concurrent`, since a busy + // io runs `async` inline, on this task. + chunked.workers.store(held + extra, .monotonic); + var started: usize = 0; + while (started < extra) : (started += 1) { + group.concurrent(self.io, ChunkedRead.run, .{&chunked}) catch break; + } + held += started; + if (started < extra) { + // The io starts no more: the rest of the permits go + // back, and this tensor stops asking. + self.permits.release(self.io, extra - started); + chunked.workers.store(held, .monotonic); + useful_workers = held; + } + } + } + if (!chunked.readOne()) break; + } + group.await(self.io) catch {}; + self.permits.release(self.io, held); + + if (chunked.failure.get()) |err| return err; + } +}; + +/// The staging one spec takes: its tensor, then room for its widest strided +/// part's source range, which is read once before its runs are copied out. +fn stagingBytes(spec: LoadSpec) usize { + var span: usize = 0; + for (spec.parts) |part| { + if (part.lattice.dims.len != 0) span = @max(span, @as(usize, @intCast(part.lattice.spans().?.src))); + } + return spec.shape.byteSize() + span; +} + +/// One buffered submission: its workers claim specs from `next`. +pub const Batch = struct { + specs: []const LoadSpec, + progress: std.Progress.Node, + next: std.atomic.Value(usize) = .init(0), + workers: std.Io.Group = .init, +}; + +/// Source reads in flight across every task of the buffered backend, at most +/// `read_parallelism`. On a high-latency source, a tensor wider than the +/// hints' `read_chunk_size` may split its read across the permits that are +/// free, for parallel streams. +const ReadPermits = struct { + mutex: std.Io.Mutex = .init, + available: std.Io.Condition = .init, + free: usize, + + fn acquire(self: *ReadPermits, io: std.Io) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (self.free == 0) self.available.waitUncancelable(io, &self.mutex); + self.free -= 1; + } + + /// Up to `count` permits, without waiting: how many were free is how much + /// concurrency this tensor gets, so a loaded source is left alone. + fn tryAcquire(self: *ReadPermits, io: std.Io, count: usize) usize { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + const taken = @min(count, self.free); + self.free -= taken; + return taken; + } + + fn release(self: *ReadPermits, io: std.Io, count: usize) void { + if (count == 0) return; + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.free += count; + for (0..count) |_| self.available.signal(io); + } +}; + +/// One tensor's read, claimed from a shared byte cursor by the owning task +/// and its helpers. Positional reads leave the file as it was, so they can +/// run concurrently on one file. +/// +/// A claim is the tensor split `workers` ways, and never smaller than the +/// hints' `read_chunk_size` -- that field is a minimum request size, and +/// on a high-latency source a request costs a round trip whether or not it +/// runs beside another. So one worker takes the whole tensor in a single +/// read, and only the concurrency the source can actually carry turns into +/// extra requests. +const ChunkedRead = struct { + io: std.Io, + source: *const safetensors.Tensor, + file: std.Io.File, + /// Where `destination` starts in the tensor's bytes. + source_offset: u64 = 0, + destination: []u8, + chunk_size: usize, + /// Reads this tensor may have in flight: the owner raises it before it + /// starts helpers on newly taken permits, and lowers it to the helpers + /// that started. + workers: std.atomic.Value(usize) = .init(1), + /// Bytes already claimed. + next: std.atomic.Value(usize) = .init(0), + failure: stdx.FirstError(safetensors.Tensor.RangeError) = .{}, + + fn run(self: *ChunkedRead) void { + while (self.readOne()) {} + } + + /// Reads the next unclaimed span. False once the tensor is fully claimed, + /// or once a read failed. + fn readOne(self: *ChunkedRead) bool { + if (self.failure.get() != null) return false; + const span = self.claim() orelse return false; + self.source.readRange(self.io, self.file, span.bytes, self.source_offset + span.offset) catch |err| { + _ = self.failure.set(err); + return false; + }; + return true; + } + + fn claim(self: *ChunkedRead) ?struct { offset: usize, bytes: []u8 } { + var start = self.next.load(.monotonic); + while (start < self.destination.len) { + const workers = self.workers.load(.monotonic); + const share = self.destination.len / workers + + @intFromBool(self.destination.len % workers != 0); + const len = @min(@max(self.chunk_size, share), self.destination.len - start); + if (self.next.cmpxchgWeak(start, start + len, .monotonic, .monotonic)) |actual| { + start = actual; + continue; + } + return .{ .offset = start, .bytes = self.destination[start..][0..len] }; + } + return null; + } +}; + +/// Host staging for the buffered backend: one budget for the buffers the +/// running tensors hold and those kept for reuse, which are handed back on +/// completion instead of freed. Mapping, faulting and then unmapping a whole +/// tensor's worth of pages for every tensor takes half of a serial buffered +/// load of a 14.96 GiB checkpoint, and a fifth of a concurrent one. +/// +/// The budget is `slots` times the largest request submitted, so no +/// request is wider than it. A request takes the smallest kept buffer that +/// fits, which counts in full while it is held: small tensors run far wider +/// than the slot count on small buffers, but once only large ones are kept +/// they take those, and run about `slots` wide. +const Staging = struct { + mutex: std.Io.Mutex = .init, + room: std.Io.Condition = .init, + /// The budget in largest requests (`staging_tensors`). + slots: usize, + /// Zero until the first submission reports its largest request. + budget: usize = 0, + /// What the running tensors hold. + live_bytes: usize = 0, + /// Buffers no tensor holds, and their total length. + free: std.ArrayList([]u8) = .empty, + retained_bytes: usize = 0, + + /// Raises the budget to `slots` of `largest`. Submissions only ever widen + /// it: a later submission of smaller tensors cannot shrink what is + /// already running. + fn widen(self: *Staging, io: std.Io, largest: usize) void { + const wanted = largest *| self.slots; + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + if (wanted <= self.budget) return; + self.budget = wanted; + self.room.broadcast(io); + } + + /// A whole allocation of at least `size` bytes, with undefined contents: + /// the allocation itself, not a slice of one. The caller slices it for + /// its own use and returns this exact slice to `release`, whatever + /// happens. Waits until a retained buffer fits, or `size` more bytes + /// fit the budget beside the live ones. `size` is at most the budget: + /// `widen` ran for the request's submission. + fn acquire(self: *Staging, allocator: std.mem.Allocator, io: std.Io, size: usize) ![]u8 { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + std.debug.assert(size <= self.budget); + while (true) { + if (self.take(size)) |buffer| return buffer; + if (self.live_bytes + size <= self.budget) break; + self.room.waitUncancelable(io, &self.mutex); + } + // No retained buffer fits, so each is smaller than this request: + // free them, smallest first, until it fits beside the rest. + while (self.live_bytes + self.retained_bytes + size > self.budget) { + const dropped = self.free.swapRemove(self.smallest()); + self.retained_bytes -= dropped.len; + allocator.free(dropped); + } + const fresh = try allocator.alloc(u8, size); + self.live_bytes += fresh.len; + std.debug.assert(self.live_bytes + self.retained_bytes <= self.budget); + return fresh; + } + + /// `buffer` must be what `acquire` returned, not a slice of it. It is + /// kept: the budget already counts it. + fn release(self: *Staging, allocator: std.mem.Allocator, io: std.Io, buffer: []u8) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.live_bytes -= buffer.len; + self.room.broadcast(io); + self.free.append(allocator, buffer) catch return allocator.free(buffer); + self.retained_bytes += buffer.len; + } + + /// Every tensor is done, so every buffer is back. + fn deinit(self: *Staging, allocator: std.mem.Allocator) void { + std.debug.assert(self.live_bytes == 0); + for (self.free.items) |buffer| allocator.free(buffer); + self.free.deinit(allocator); + } + + /// The smallest retained buffer that fits, now live, so the big ones + /// stay free for the big tensors. Called with `mutex` held. + fn take(self: *Staging, size: usize) ?[]u8 { + var best: ?usize = null; + for (self.free.items, 0..) |buffer, i| { + if (buffer.len < size) continue; + if (best == null or buffer.len < self.free.items[best.?].len) best = i; + } + const buffer = self.free.swapRemove(best orelse return null); + self.retained_bytes -= buffer.len; + self.live_bytes += buffer.len; + return buffer; + } + + /// Called with `mutex` held and a buffer retained. + fn smallest(self: *const Staging) usize { + var index: usize = 0; + for (self.free.items, 0..) |buffer, i| { + if (buffer.len < self.free.items[index].len) index = i; + } + return index; + } +}; + +test "chunked read splits a tensor only as wide as it has workers" { + var destination: [1000]u8 = undefined; + var work: ChunkedRead = .{ + .io = undefined, + .source = undefined, + .file = undefined, + .destination = &destination, + .chunk_size = 100, + }; + + // One worker reads the whole tensor, exactly as an unsplit read would. + const whole = work.claim().?; + try std.testing.expectEqual(@as(usize, 0), whole.offset); + try std.testing.expectEqual(@as(usize, 1000), whole.bytes.len); + try std.testing.expect(work.claim() == null); + + // Four workers take a quarter each. + work.next = .init(0); + work.workers = .init(4); + for (0..4) |i| { + const span = work.claim().?; + try std.testing.expectEqual(i * 250, span.offset); + try std.testing.expectEqual(@as(usize, 250), span.bytes.len); + } + try std.testing.expect(work.claim() == null); + + // Never below the read chunk, however many workers there are. + work.next = .init(0); + work.workers = .init(50); + const floored = work.claim().?; + try std.testing.expectEqual(@as(usize, 100), floored.bytes.len); +} + +test "staging holds live and retained bytes within one budget" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var staging: Staging = .{ .slots = 3 }; + defer staging.deinit(allocator); + + // Three slots of the largest request is the budget, and a later, + // smaller submission cannot shrink it. + staging.widen(io, 16); + staging.widen(io, 4); + try std.testing.expectEqual(@as(usize, 48), staging.budget); + + // With nothing kept, small requests run far wider than the slot count, + // and their buffers are kept. + var small: [12][]u8 = undefined; + for (&small) |*buffer| buffer.* = try staging.acquire(allocator, io, 4); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + for (small) |buffer| staging.release(allocator, io, buffer); + try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); + + // A request no retained buffer fits frees as many as it needs room + // from. + const large = try staging.acquire(allocator, io, 16); + try std.testing.expectEqual(@as(usize, 32), staging.retained_bytes); + staging.release(allocator, io, large); + // The smallest buffer that fits comes back to a smaller request, so the + // 16 stays for one the 4s cannot serve. The first take moves the 16 to + // the front of the list, ahead of the second's 4. + var reused: [2][]u8 = undefined; + for (&reused) |*buffer| { + buffer.* = try staging.acquire(allocator, io, 3); + try std.testing.expectEqual(@as(usize, 4), buffer.len); + } + try std.testing.expectEqual(@as(usize, 40), staging.retained_bytes); + for (reused) |buffer| staging.release(allocator, io, buffer); + try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); + + // A full budget holds the next request until a buffer comes back, then + // hands it that buffer. + var held: [3][]u8 = undefined; + for (&held) |*buffer| buffer.* = try staging.acquire(allocator, io, 16); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes + staging.retained_bytes); + var waiting = try io.concurrent(Staging.acquire, .{ &staging, allocator, io, 16 }); + try io.sleep(.fromMilliseconds(5), .awake); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + staging.release(allocator, io, held[0]); + held[0] = try waiting.await(io); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + for (held) |buffer| staging.release(allocator, io, buffer); + + // Once only large buffers are kept, small requests take them, each + // counted in full: three fill the budget. + var on_large: [3][]u8 = undefined; + for (&on_large) |*buffer| buffer.* = try staging.acquire(allocator, io, 4); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + for (on_large) |buffer| staging.release(allocator, io, buffer); + try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); +} + +test "buffered loader assembles a composite destination in reused staging" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 2); + defer devices.deinit(); + const platform = devices.platform; + const mesh = try devices.mesh("tp"); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var a: [48]u8 = undefined; + testing.pattern(&a, 1); + var b: [32]u8 = undefined; + testing.pattern(&b, 2); + var full: [128]u8 = undefined; + testing.pattern(&full, 3); + var big: [256]u8 = undefined; + testing.pattern(&big, 4); + try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &a }, + .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &b }, + .{ .name = "full", .dtype = "U16", .dims = &.{ 8, 8 }, .bytes = &full }, + .{ .name = "big", .dtype = "U16", .dims = &.{ 16, 8 }, .bytes = &big }, + }); + // Rows split over the two devices: b at row 1, a's last two rows (from + // byte 16 of a) at row 4, and a strided part, the left halves of full's + // rows 0 and 2, at rows 6 and 7, around a leading row, a gap and the + // right halves of zeros. + const shape = Shape.init(.{ .row = 8, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); + const columns = [_]backend.Lattice.Dim{.{ .count = 2, .src_stride = 32, .dst_stride = 16 }}; + const parts = [_]LoadPart{ + .{ .source = registry.tensors.getPtr("b").?, .lattice = .{ .dst_offset = 16, .width = 32 } }, + .{ .source = registry.tensors.getPtr("a").?, .lattice = .{ .src_offset = 16, .dst_offset = 64, .width = 32 } }, + .{ .source = registry.tensors.getPtr("full").?, .lattice = .{ .dst_offset = 96, .width = 8, .dims = &columns } }, + }; + try std.testing.expect(backend.checkParts(&parts, shape.byteSize())); + var expected: [128]u8 = @splat(0); + @memcpy(expected[16..48], &b); + @memcpy(expected[64..96], a[16..48]); + @memcpy(expected[96..104], full[0..8]); + @memcpy(expected[112..120], full[32..40]); + const whole = [_]LoadPart{.whole(registry.tensors.getPtr("full").?)}; + const big_shape = Shape.init(.{ .row = 16, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); + const big_whole = [_]LoadPart{.whole(registry.tensors.getPtr("big").?)}; + + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.destroy(); + // A bigger tensor first, so the composite, its strided source range + // included, reuses its staging buffer and only the zeroing keeps its + // bytes out of the gaps. + var first: Buffer = undefined; + try loader.awaitBatch(try loader.submit(&.{.{ .shape = big_shape, .output = &first, .parts = &big_whole }}, .none)); + defer first.deinit(); + try testing.expectShards(allocator, io, &first, &big, .default); + var output: Buffer = undefined; + try loader.awaitBatch(try loader.submit(&.{.{ .shape = shape, .output = &output, .parts = &parts }}, .none)); + defer output.deinit(); + try testing.expectShards(allocator, io, &output, &expected, .default); + // One buffer was ever allocated, within the budget of two slots of the + // largest request. + try std.testing.expectEqual(@as(usize, 512), loader.staging.budget); + try std.testing.expectEqual(@as(usize, 256), loader.staging.retained_bytes); + + // A source whose file does not exist fails the worker, and the error + // is sticky from then on. + var missing = registry.tensors.get("full").?; + missing.file_uri = "missing.safetensors"; + const missing_parts = [_]LoadPart{.whole(&missing)}; + var never_written: Buffer = undefined; + const batch = try loader.submit(&.{.{ .shape = shape, .output = &never_written, .parts = &missing_parts }}, .none); + try std.testing.expectError(error.FileNotFound, loader.awaitBatch(batch)); + // `expectError` would print the batch, and every type its specs name. + if (loader.submit(&.{.{ .shape = shape, .output = &never_written, .parts = &whole }}, .none)) |_| + return error.TestUnexpectedResult + else |err| + try std.testing.expectEqual(error.FileNotFound, err); +} + +test "a worker waiting for its staging loads nothing once the loader failed" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var bytes: [64]u8 = undefined; + testing.pattern(&bytes, 1); + try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "x", .dtype = "U8", .dims = &.{ 8, 8 }, .bytes = &bytes }, + }); + const shape = Shape.init(.{ .row = 8, .col = 8 }, .u8); + const parts = [_]LoadPart{.whole(registry.tensors.getPtr("x").?)}; + + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 1 }); + defer loader.destroy(); + // The test holds the whole budget, one slot of the tensor, so the + // worker waits for its staging until the loader failed. + loader.staging.widen(io, bytes.len); + const held = try loader.staging.acquire(allocator, io, bytes.len); + var output: Buffer = .{ ._platform = devices.platform, ._shape = shape, ._shards = .empty }; + defer output.deinit(); + const batch = try loader.submit(&.{.{ .shape = shape, .output = &output, .parts = &parts }}, .none); + try io.sleep(.fromMilliseconds(5), .awake); + loader.abort(); + loader.staging.release(allocator, io, held); + try std.testing.expectError(error.Canceled, loader.awaitBatch(batch)); + try std.testing.expectEqual(@as(usize, 0), output._shards.len); +} + +test "an aborted loader leaves each output as it was or whole, and stays failed" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var bytes: [64]u8 = undefined; + testing.pattern(&bytes, 1); + try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "x", .dtype = "U8", .dims = &.{ 8, 8 }, .bytes = &bytes }, + }); + const shape = Shape.init(.{ .row = 8, .col = 8 }, .u8); + const parts = [_]LoadPart{.whole(registry.tensors.getPtr("x").?)}; + // One worker over many specs, so the abort lands among them. + var specs: [64]LoadSpec = undefined; + var outputs: [64]Buffer = undefined; + for (&specs, &outputs) |*spec, *output| { + output.* = .{ ._platform = devices.platform, ._shape = shape, ._shards = .empty }; + spec.* = .{ .shape = shape, .output = output, .parts = &parts }; + } + defer for (&outputs) |*output| output.deinit(); + + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 1 }); + defer loader.destroy(); + const batch = try loader.submit(&specs, .none); + loader.abort(); + try std.testing.expectError(error.Canceled, loader.awaitBatch(batch)); + for (&outputs) |*output| { + if (output._shards.len != 0) try testing.expectShards(allocator, io, output, &bytes, .default); + } + if (loader.submit(&specs, .none)) |_| + return error.TestUnexpectedResult + else |err| + try std.testing.expectEqual(error.Canceled, err); +} + +test "buffered loader runs overlapping batches over their cursors, with or without concurrency" { + const allocator = std.testing.allocator; + var devices: testing.Devices = undefined; + try devices.init(allocator, std.testing.io, 2); + defer devices.deinit(); + const mesh = try devices.mesh("tp"); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + // Tensors of 2 to 32 rows of 8 u16, so the staging budget holds a + // varying number of them. + const count = 24; + var bytes: [count][512]u8 = undefined; + var names: [count][8]u8 = undefined; + var dims: [count][2]i64 = undefined; + var entries: [count]testing.TensorBytes = undefined; + for (&bytes, &names, &dims, &entries, 0..) |*data, *name, *dim, *entry, i| { + const rows = 2 * (i % 16 + 1); + testing.pattern(data[0 .. rows * 16], @intCast(i + 1)); + dim.* = .{ @intCast(rows), 8 }; + entry.* = .{ .name = try std.fmt.bufPrint(name, "t{d}", .{i}), .dtype = "U16", .dims = dim, .bytes = data[0 .. rows * 16] }; + } + try testing.writeSafetensors(allocator, std.testing.io, tmp.dir, "weights.safetensors", ®istry, &entries); + + var parts: [count][1]LoadPart = undefined; + var specs: [count]LoadSpec = undefined; + var outputs: [count]Buffer = undefined; + for (&parts, &specs, &outputs, entries) |*part, *spec, *output, entry| { + part[0] = .whole(registry.tensors.getPtr(entry.name).?); + const shape = Shape.init(.{ .row = entry.dims[0], .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); + spec.* = .{ .shape = shape, .output = output, .parts = part }; + } + + var no_concurrency: std.Io.Threaded = .init(allocator, .{ .async_limit = .nothing, .concurrent_limit = .nothing }); + defer no_concurrency.deinit(); + for ([_]std.Io{ std.testing.io, no_concurrency.io() }) |io| { + // A high-latency source with 32-byte chunks: helpers split each + // tensor whenever a permit is free. + const loader = try Loader.create(allocator, io, devices.platform, .{ + .read_parallelism = 4, + .read_hints = .{ .read_chunk_size = 32, .high_latency = true }, + }); + defer loader.destroy(); + // Both batches are in flight before either is awaited, each with + // its own workers: the permits bound their reads and the staging + // budget their bytes (`Staging.acquire` asserts it). + const first = try loader.submit(specs[0 .. count / 2], .none); + const second = try loader.submit(specs[count / 2 ..], .none); + try loader.awaitBatch(first); + try loader.awaitBatch(second); + defer for (&outputs) |*output| output.deinit(); + for (&outputs, entries) |*output, entry| try testing.expectShards(allocator, io, output, entry.bytes, .default); + try std.testing.expect(loader.staging.retained_bytes <= loader.staging.budget); + try std.testing.expectEqual(@as(usize, 0), loader.staging.live_bytes); + try std.testing.expectEqual(@as(usize, 4), loader.permits.free); + } +} + +test "a tensor from a high-latency HTTP source splits its read over the free permits" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + + var object: [4096]u8 = undefined; + testing.pattern(&object, 1); + var server = try VFS.MockServer.init(io, &object, .{}); + defer server.deinit(io); + try server.start(io); + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var http: VFS.HTTP = try .init(allocator, io, &client, .{ .protocol = .http }); + defer http.deinit(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + try vfs.registerBackend("http", http.backend()); + + var url_buffer: [64]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + const source: safetensors.Tensor = .{ .file_uri = url, .name = "x", .shape = .init(.{object.len}, .u8), .offset = 0 }; + const parts = [_]LoadPart{.whole(&source)}; + + // Four free permits and a quarter of the tensor as the read chunk: the + // worker and three helpers each read a quarter, one range GET each. + const loader = try Loader.create(allocator, vfs.io(), devices.platform, .{ + .read_parallelism = 4, + .read_hints = .{ .read_chunk_size = object.len / 4, .high_latency = true }, + }); + defer loader.destroy(); + var output: Buffer = undefined; + try loader.awaitBatch(try loader.submit(&.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, .none)); + defer output.deinit(); + try server.stop(io); + try testing.expectShards(allocator, io, &output, &object, .default); + try std.testing.expectEqual(@as(usize, 4), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 4), loader.permits.free); +} diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig new file mode 100644 index 00000000..a68e883b --- /dev/null +++ b/zml/io/direct_loader.zig @@ -0,0 +1,5361 @@ +//! Direct loading follows a batch from per-file planning through FIFO claims, +//! source reads and DMA completion. Runtime ownership and synchronization +//! live here. CUDA, ROCm, oneAPI, Furiosa and CPU load this way +//! (`backend.loadsDirect`). +//! +//! Sizing, fixed by `Loader.create`. Calibration picks the DMA block +//! (dma_calibration.zig) and maps the ring it measures in; each device's +//! DMA budget is `dma_calibration.depth` blocks. A source request is the +//! checkpoint's minimum read (`VFS.readHints`) rounded up to whole blocks, +//! one block at least (`backend.requestSize`). The source width is +//! `read_parallelism`, clipped to what the mapped ceiling fits. The +//! workspace then maps, once and before the first read, what the ring +//! leaves of `width + 1` requests, the DMA reserve and the zero block, and +//! cuts them into blocks (`Workspace.carve`): nothing maps during a load. +//! `width + 1` workers share `width` read permits, so a full width of reads +//! is in flight while one worker queues what it read. The lifecycle credits +//! are the whole pinned set in requests, so transfers that still hold +//! blocks keep the DMA stage fed and never want a block the reads did not +//! leave free. +//! +//! Planning and claims. `submit` sorts a submission's parts by file and +//! offset, plans one file at a time and publishes each plan at once, so the +//! workers read the first file while the next is planned. A plan merges +//! touching or overlapping ranges into reads, never across a gap or a file, +//! and cuts them into jobs of at most a request; jobs keep that file order, +//! and the `Scheduler` hands them out in submission order. A worker takes a +//! credit, claims a job, reads it into its slot's pinned blocks and queues +//! its preplanned transfer pieces on per-device pumps. A block may feed +//! several outputs or devices; the request keeps its credit until every +//! piece it queued completed or was abandoned. The last-transfer flag goes +//! on the piece that completes a destination's bytes, so reads and DMA +//! complete in any order. A batch owns its plans and callback contexts +//! until every job completes; a failure retires the unclaimed jobs and +//! drains what is in flight before the batch goes. +//! +//! Direct I/O. A local file the VFS reads past the page cache +//! (`VFS.useDirectIo`, asked once per file: `Planner.Reads`) is read in +//! whole alignment units: each job's read widens to aligned bounds around +//! its range, into blocks that are whole units, and its transfers address +//! the widened read. A job's range is at most a request less two units, so +//! its widened read still fits its credit's slot. Every other file is read +//! exact. The arenas are placed for how the checkpoint reads, predicted +//! before calibration maps the first (`Planner.Reads.predict`): over every +//! allowed node when at least four fifths of its bytes will read direct +//! (host_memory.zig). The files predicted then read as predicted, and every +//! other file as its own decision says. +//! +//! Destinations. A part's source bytes map to its destination's logical +//! offsets, which the destination's `ShardGrid` places on the devices by +//! arithmetic. Pieces of one destination merge across parts when block, +//! writers and both offsets are contiguous, and its parts may come from +//! different files and plans. Bytes no part covers go out with the plan of +//! its first part, as a last fill job that reads nothing and transfers from +//! the zero block, which no credit owns. The job of a tensor sharded along +//! any axis but its first is staged by destination where that saves +//! submissions (`JobCut.placeDma`). Strided parts are gathered +//! (`JobCut.appendGatherJobs`), their runs copied out of the worker's bounce +//! buffer by `Lattice.copyRuns`. Block-scale swizzles gather this way, as +//! runs of four bytes. Runs under 4 KiB cost host copies, where a program +//! costs a compile and staging memory: on DeepSeek-V4.1 over two GB300, +//! laying the MXFP4 experts out here (gate and up rows interleaved, scales +//! swizzled) copied 77 to 189 GiB through bounce buffers, 17 to 36 s of +//! worker time; as programs it copied 1 to 3 GiB and loaded 0.1 to 1.1 s +//! faster, for 4 to 6 GiB more device memory at the peak. +//! +//! Host memory. A destination in a host memory space (`Memory.isHost`) +//! takes no DMA: the worker that reads its bytes writes them in place into +//! its PJRT host buffers, which an allocation task creates ahead of the +//! reads (`Loader.startAllocations`). A host job reads straight into its +//! destinations when its read is exact and addresses each byte once, in +//! order; a widened read, or parts that overlap, goes through its slot's +//! blocks and is copied from there. Bytes no part covers are zeroed in +//! place. The devices holding one box share one allocation, through views +//! of it (`SharedHostAllocation`), so a replicated destination is written +//! once. A `host_mapped` destination's shards are always such views, in +//! the devices' default memory space, so kernels read the pinned memory in +//! place. Host and DMA parts never share a job, and no strided part goes +//! to host memory. +//! +//! Locks. Where several pumps are held, they are taken in device order. A +//! pump's mutex may be held while a released request gives its credit back +//! (`Credits`), never the reverse, and no other lock is taken under the +//! scheduler's mutex. A DMA callback, on a plugin thread, holds one pump at +//! a time. + +const std = @import("std"); +const builtin = @import("builtin"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); +const VFS = @import("vfs"); +const zio = @import("zio"); + +const Buffer = @import("../buffer.zig").Buffer; +const pjrtx = @import("../pjrtx.zig"); +const Platform = @import("../platform.zig").Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const backend = @import("backend.zig"); +const BackendOptions = backend.Options; +const LoadPart = backend.LoadPart; +const LoadSpec = backend.LoadSpec; +const ShardGrid = @import("ShardGrid.zig"); +const dma_calibration = @import("dma_calibration.zig"); +const Memory = @import("TensorStore.zig").Memory; +const host_memory = @import("host_memory.zig"); +const load_log = @import("log.zig").load; +const testing = @import("../testing.zig"); + +/// Creating a tensor's device state: host memory or PJRT, since the front +/// end checked its placement and memory kind. `LazyOnce` replays it to every +/// later reader of the tensor. +const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; + +/// Bounds per-device event overhead for tiny tensors. This is not a measured +/// optimum: 64 pieces smaller than block_size / 8 cannot fill an eight-block +/// byte budget. +const max_dma_pieces_per_device: usize = 64; + +/// The direct DMA backend. Submissions and awaits come from one task at a +/// time; the workers and the pumps run concurrently with them. +pub const Loader = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + /// The pinned arenas, grown once at creation and cut into `blocks`. + workspace: host_memory.Workspace, + /// The workspace's blocks (`carve`): the zero block first, then the + /// credits' slots (`Credits`). The zero block is the source of every DMA + /// fill transfer, in no credit's slot, never written again. + blocks: []const host_memory.Block, + scheduler: Scheduler, + metrics: Metrics = .{}, + /// One permit per source read in flight (`readSlot`), `read_width` of + /// them. The width never changes: a source that answers 429 (or 503 on + /// the object stores) is held one layer down, where the VFS holds every + /// request of that backend (`vfs/request.zig`), so the traffic falls + /// while the memory bounds stay. + read_permits: std.Io.Semaphore, + read_width: usize, + /// Lifecycle credits (`Credits`), beyond the workers' through the DMA + /// reserve. One credit beyond the width fed the DMA engines one request + /// at a time: on a GB300 loading DeepSeek-V4 (pieces of 4 MiB and 256 + /// KiB) that was 24.3 GiB/s at width 16 against 43.8 GiB/s with eight + /// requests queued, and the workers spent 9 ms per request waiting for a + /// credit against 3 ms reading. + credits: Credits, + /// The sticky first error (`recordError`). + first_error: stdx.FirstError(anyerror) = .{}, + /// One per device (`Dma.DevicePump`). Where several are locked at once + /// (`Dma.enqueue`, `Dma.retireBatch`) they are taken in device + /// order; a pump, a completion and `Dma.abortReady` hold one at a time. + pumps: []Dma.DevicePump, + /// Per-device in-flight budget: `dma_calibration.depth` blocks, in bytes. + dma_budget_bytes: usize, + worker_group: std.Io.Group = .init, + /// One per source path. Its file opens at first use (during planning + /// when a VFS decides direct IO for a local file, otherwise at the first + /// read) and stays open until `destroy`. Keyed by the slot's own copy of + /// the path: the registry that named it may go once the batch retired. + source_slots: std.StringHashMapUnmanaged(*SourceSlot) = .empty, + /// Device bytes of the outputs of the sealed batches, per + /// `platform.devices` index, cumulative, as `DestinationState.init` + /// allocates them; counted by the submitting task. + sealed_bytes: backend.DeviceBytes = @splat(0), + /// Device bytes allocated for outputs so far, per `platform.devices` + /// index, cumulative. Freeing an output does not decrease them. + allocated_bytes: [Platform.MAX_NUM_DEVICES]std.atomic.Value(u64) = @splat(.init(0)), + /// What the debug lines time from (`logBatch`, `logSummary`). + created_at: std.Io.Timestamp, + /// Submissions so far; the next batch's sequence number. + batch_count: usize = 0, + /// The block size, the request size and the gather limit, for the whole + /// load. + plan_config: Planner.Config, + /// How each file is read: exact, or aligned for direct I/O. + reads: Planner.Reads, + + pub fn create( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: BackendOptions, + ) !*Loader { + // A destination's shards and a transfer's writers (a u64 mask) are + // per device; Platform refuses devices past its maximum. + comptime std.debug.assert(Platform.MAX_NUM_DEVICES <= ShardGrid.max_boxes); + const self = try allocator.create(Loader); + errdefer allocator.destroy(self); + + var reads: Planner.Reads = .init(io, opts.direct_io); + errdefer reads.deinit(allocator); + // The arenas are placed for how the checkpoint reads before + // calibration maps the first: registered pages never move. + const probe_started: std.Io.Timestamp = .now(io, .awake); + const predicted = switch (platform.io_impl) { + .threaded => try reads.predict(allocator, opts.checkpoint), + // zio hands a file operation its event loop does not run (a + // real path; any, where it falls back to epoll) to a pool + // worker, and the compiles hold theirs for their whole run + // (`Compiler.compile`): the prediction takes a worker of its own + // rather than queue behind them, so the set-up overlaps them. + .zio => try zio.blockInPlaceReserved(Planner.Reads.predict, .{ &reads, allocator, opts.checkpoint }), + }; + if (opts.checkpoint != null) { + const probe_ns: u64 = @intCast(@max(probe_started.untilNow(io, .awake).nanoseconds, 0)); + load_log.debug("checkpoint reads predicted: files={d}, direct={Bi:.2} of {Bi:.2}, direct_reads={}, probe_ms={d:.3}", .{ + predicted.files, + predicted.direct_bytes, + predicted.bytes, + predicted.direct(), + @as(f64, @floatFromInt(probe_ns)) / std.time.ns_per_ms, + }); + } + var workspace: host_memory.Workspace = try .init(allocator, io, platform, predicted.direct()); + errdefer workspace.deinit(); + // At debug when canceled (`zml.io.Loader.deinit` cancels a set-up + // no `start` waited for) or out of memory (the workspace logs what + // it fails to map or pin). + const block_size = dma_calibration.calibrate(&workspace, platform, .{ .block_sizes = opts.dmaBlockSizes() }) catch |err| { + load_log.failure(err, "calibrate DMA: target={t}", .{platform.target}); + return err; + }; + const request_size = backend.requestSize(opts.read_hints.read_chunk_size, block_size, reads.alignment).?; + const blocks_per_request = request_size / block_size; + // The DMA stage of every device, kept mapped beside the source + // working set; at least one maximal source request, so a single + // request always fits beside it. + const dma_reserve = @max( + dma_calibration.depth * platform.devices.len, + Planner.blockCount(backend.max_read_request_size, block_size), + ); + // One more block for the zero fill, which no credit owns. + const zero_blocks = 1; + const available = workspace.capacity(block_size); + const width = @min( + opts.read_parallelism, + (available -| dma_reserve -| zero_blocks) / blocks_per_request -| 1, + ); + if (width == 0) { + load_log.err("host memory ceiling {Bi:.2} fits no source request beside the DMA reserve: available_blocks={d}, reserve_blocks={d}, blocks_per_request={d}: OutOfMemory", .{ host_memory.Workspace.mapped_bytes_ceiling, available, dma_reserve, blocks_per_request }); + return error.OutOfMemory; + } + if (width < opts.read_parallelism) { + load_log.debug("DMA source working set clipped by the mapped ceiling: width={d} of {d}", .{ + width, + opts.read_parallelism, + }); + } + // Two arenas, the reserve first: ROCm balances bytes across its host + // nodes per allocation, and that split is recorded. + const pregrowth_started: std.Io.Timestamp = .now(io, .awake); + const retained_before = workspace.mappedBytes(); + // The workspace logs what it fails to map. + const blocks = try workspace.carve(block_size, dma_reserve, (width + 1) * blocks_per_request + dma_reserve + zero_blocks); + const pregrowth_ns: u64 = @intCast(@max(pregrowth_started.untilNow(io, .awake).nanoseconds, 0)); + load_log.debug("host workspace pregrown: retained={Bi:.2}, pregrown={Bi:.2}, pregrowth_ms={d:.3}", .{ + workspace.mappedBytes(), + workspace.mappedBytes() - retained_before, + @as(f64, @floatFromInt(pregrowth_ns)) / std.time.ns_per_ms, + }); + std.debug.assert(blocks.len != 0); + @memset(blocks[0], 0); + var credits: Credits = try .init(allocator, blocks[1..], blocks_per_request); + errdefer credits.deinit(allocator); + const workers = width + 1; + std.debug.assert(credits.count() >= workers); + const pumps = try allocator.alloc(Dma.DevicePump, platform.devices.len); + errdefer allocator.free(pumps); + @memset(pumps, .{}); + + self.* = .{ + .allocator = allocator, + .io = io, + .platform = platform, + .workspace = workspace, + .blocks = blocks, + .scheduler = .init(allocator), + .read_permits = .{ .permits = width }, + .read_width = width, + .credits = credits, + .pumps = pumps, + .dma_budget_bytes = dma_calibration.depth * block_size, + .created_at = .now(io, .awake), + .plan_config = .{ + .block_size = block_size, + .request_size = request_size, + .gather_limit = backend.requestSize(opts.read_hints.read_chunk_size, std.mem.min(usize, opts.dmaBlockSizes()), reads.alignment).?, + }, + .reads = reads, + }; + std.debug.assert(self.dma_budget_bytes > 0); + errdefer self.scheduler.deinit(); + + errdefer self.stopWorkers(); + for (0..workers) |_| self.worker_group.concurrent(io, workerMain, .{self}) catch |err| { + load_log.err("start {d} source workers: {s}", .{ workers, @errorName(err) }); + return err; + }; + load_log.debug("live loader ready: target={s}, request_size={Bi:.2}, direct_io={t}, source_alignment={d}, dma_block_size={Bi:.2}, dma_budget_per_device={Bi:.2}, source_width={d}, lifecycle_credits={d}, workers={d}, retained={Bi:.2}", .{ + @tagName(platform.target), + request_size, + opts.direct_io, + reads.alignment, + block_size, + self.dma_budget_bytes, + width, + self.credits.count(), + workers, + self.workspace.mappedBytes(), + }); + return self; + } + + /// Plans `specs` one source file at a time, publishes each file's plan + /// as soon as it exists behind every earlier submission, then seals the + /// batch: work on the first file starts while the rest is planned. The + /// batch owns its destinations and parts until `awaitBatch` retires it. + /// The specs keep the `LoadSpec` contract and `stridedRead`'s limit, + /// which the front end checked. A planning failure fails the loader (a + /// partial submission can never complete): the batch is awaited here + /// and the caller sees only the first error. A failed loader refuses + /// every submit with its first error. The batch's host-resident + /// destinations are then created ahead of their reads + /// (`startAllocations`). + pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { + try self.checkOpen(); + const batch = batch: { + const batch = try Batch.create(self.allocator, self.io, self.batch_count); + errdefer batch.destroy(); + try self.createDestinations(batch, specs, progress); + break :batch batch; + }; + Planner.publishFiles(&self.scheduler, self.io, batch, batch.parts, self.plan_config, self.reads) catch |err| { + // Fail the loader, seal and await what was published of this + // batch, and return the loader's first error. + self.recordError(err, "plan batch {d}, published_plans={d}", .{ batch.sequence, batch.plans.items.len }); + batch.sealed_at = .now(self.io, .awake); + self.batch_count += 1; + batch.releaseUnits(1); + try self.awaitBatch(batch); + unreachable; + }; + // Stamped before the sentinel goes, so before the batch can complete. + batch.sealed_at = .now(self.io, .awake); + self.batch_count += 1; + self.countSealed(batch); + self.startAllocations(batch); + // Every plan is visible: drop the publish sentinel. A batch without + // jobs completes right here. + batch.releaseUnits(1); + return batch; + } + + /// The source bytes every job of `part` reads whole, against one read + /// (`Planner.stridedRead`). + pub fn stridedRead(self: *const Loader, part: LoadPart) backend.Backend.StridedRead { + return Planner.stridedRead(self.plan_config, self.reads, part); + } + + /// Creates the batch's host-resident destinations ahead of their reads, + /// in claim order, on a task of its own. A first-time pinned allocation + /// runs at about 14-15 GiB/s on four B70 (oneAPI), serialized whatever + /// the number of callers (one to sixteen threads measured alike; the + /// time is the xe driver's VM_BIND), so on the workers' read path it + /// stalled them for 10-12 ms per tensor; ahead of them it overlaps the + /// reads. A worker that still finds its destination missing creates it + /// itself (`LazyOnce`). The task holds one completion unit, so the batch + /// outlives it; without a task the workers create everything. + fn startAllocations(self: *Loader, batch: *Batch) void { + var host_destinations: usize = 0; + for (batch.destinations) |*destination| host_destinations += @intFromBool(destination.memory.isHost()); + if (host_destinations == 0) return; + _ = batch.remaining.fetchAdd(1, .acq_rel); + self.worker_group.concurrent(self.io, allocateMain, .{ self, batch, host_destinations }) catch |err| { + load_log.debug("batch {d}: no allocation task, workers create its {d} host destinations: {s}", .{ batch.sequence, host_destinations, @errorName(err) }); + batch.releaseUnits(1); + }; + } + + /// Walks the host jobs in claim order (the plans in publish order, each + /// plan's jobs in order) and their transfers as `run` does, skipping a + /// destination already created. `ensured` also counts one a worker was + /// still creating, which the task waited for. + fn allocateMain(self: *Loader, batch: *Batch, host_destinations: usize) void { + var ensured: usize = 0; + var previous: ?*Destination = null; + walk: for (batch.plans.items) |plan| for (plan.jobs) |job| { + switch (job.kind) { + .host_direct, .host_bounce, .host_fill => {}, + .dma_read, .dma_land, .dma_fill => continue, + } + for (plan.transfersOf(job)) |transfer| { + const destination = transfer.destination; + if (destination == previous or destination.state.readyValue() != null) continue; + previous = destination; + if (self.first_error.get() != null) break :walk; + const started = awakeNs(self.io); + _ = destination.ensureState(self) catch |err| { + self.recordError(err, "initialize tensor {s} from {s} ahead of its reads, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); + break :walk; + }; + _ = self.metrics.ahead_init_ns.fetchAdd(awakeNs(self.io) -| started, .monotonic); + ensured += 1; + } + }; + load_log.debug("batch allocations: batch={d}, host_destinations={d}, ensured={d}, done=+{d:.3}s", .{ batch.sequence, host_destinations, ensured, secondsBetween(self.created_at, .now(self.io, .awake)) }); + batch.releaseUnits(1); + } + + /// Waits for the batch's last completion unit, retires it and returns + /// the loader's sticky error if the loader failed. Destroying the batch + /// destroys its transfer managers, and PJRT's manager destructor errors + /// every buffer whose last transfer never went out, so a buffer the + /// failure left open reports that error, never success. A transfer + /// that fails asynchronously errors its buffer's definition event too + /// (the done event carries no error). A program reading such a buffer + /// fails its execution; a bulk buffer shows the error at its first use. + /// Either way an output of a failed submission holds buffers whose + /// contents are unspecified, or was never written. A batch that + /// completed without an error submitted every byte of every target (the + /// pump flagged the submission that completed its bytes); one that did + /// not would leave a buffer PJRT then errors, a bookkeeping bug this + /// panics on. A host-resident destination has no target: its workers + /// write it in place, which only the debug check of its progress covers. + pub fn awaitBatch(self: *Loader, batch: *Batch) !void { + batch.done.waitUncancelable(self.io); + const done_at: std.Io.Timestamp = .now(self.io, .awake); + // Every request of this batch has completed: no worker or callback + // touches its managers or contexts any more. + const load_error = self.first_error.get(); + if (load_error == null) for (batch.destinations) |*destination| { + const submitted = submitted: { + const state = destination.state.readyValue() orelse break :submitted false; + for (state.targets) |target| { + if (!target.fullySubmitted()) break :submitted false; + } + break :submitted true; + }; + if (!submitted) std.debug.panic("loader: batch {d} completed without submitting every byte of tensor {s} {f}", .{ batch.sequence, destination.name(), destination.shape }); + }; + // Each destination completed its progress item. + if (load_error == null) for (batch.destinations) |*destination| std.debug.assert(destination.unread_bytes.load(.acquire) == 0); + Dma.retireBatch(self, batch); + self.logBatch(batch, done_at, load_error == null); + self.destroyBatch(batch); + if (load_error) |err| return err; + } + + /// Fails the loader unless it already failed (`recordError`): unclaimed + /// jobs retire, queued pieces drain, and requests in flight finish + /// their read. + pub fn abort(self: *Loader) void { + self.recordError(error.Canceled, "loader aborted", .{}); + } + + /// The caller awaited every batch before this; nothing is queued or in + /// flight, so stopping the workers is a plain shutdown. + pub fn destroy(self: *Loader) void { + self.stopWorkers(); + self.logSummary(); + var slots = self.source_slots.valueIterator(); + while (slots.next()) |slot| { + slot.*.deinit(self.io); + self.allocator.free(slot.*.uri); + self.allocator.destroy(slot.*); + } + self.source_slots.deinit(self.allocator); + self.reads.deinit(self.allocator); + std.debug.assert(self.credits.inUse(self.io) == 0); + for (self.pumps) |*device_pump| { + std.debug.assert(device_pump.active_pieces == 0); + std.debug.assert(device_pump.queue.len == 0); + std.debug.assert(device_pump.retired == null); + device_pump.queue.deinit(self.allocator); + } + self.allocator.free(self.pumps); + self.scheduler.deinit(); + self.credits.deinit(self.allocator); + self.workspace.deinit(); + self.allocator.destroy(self); + } + + /// Output bytes of the sealed batches not allocated yet, per + /// `platform.devices` index. Workers allocate as they read, possibly + /// before the batch is sealed, so the difference saturates. + pub fn unallocatedBytes(self: *const Loader) backend.DeviceBytes { + var unallocated = self.sealed_bytes; + for (&unallocated, &self.allocated_bytes) |*bytes, *allocated| bytes.* -|= allocated.load(.acquire); + return unallocated; + } + + /// Counts the device bytes of the sealed `batch`: a box of each + /// destination outside host memory on each of its devices, as + /// `DestinationState.init` allocates them. + fn countSealed(self: *Loader, batch: *const Batch) void { + for (batch.destinations) |*destination| { + if (destination.memory.isHost()) continue; + for (destination.grid.devices) |device| self.sealed_bytes[device.id] += destination.grid.box_bytes; + } + } + + fn checkOpen(self: *Loader) !void { + if (self.first_error.get()) |err| return err; + } + + /// Creates the batch's destinations, one per spec, and their parts in + /// its arena, their sources and dimensions staying the specs' until the + /// batch is awaited (`backend.LoadSpec`). A destination's device state + /// is created lazily by the first worker that reads or fills for it, + /// or by the allocation task. + fn createDestinations( + self: *Loader, + batch: *Batch, + specs: []const LoadSpec, + progress: std.Progress.Node, + ) !void { + const arena = batch.arena.allocator(); + var part_count: usize = 0; + for (specs) |spec| part_count += spec.parts.len; + const destinations = try arena.alloc(Destination, specs.len); + const parts = try arena.alloc(Part, part_count); + var next_part: usize = 0; + for (specs, destinations) |spec, *destination| { + const grid: ShardGrid = .init(self.platform, spec.shape); + const destination_parts = parts[next_part..][0..spec.parts.len]; + next_part += spec.parts.len; + var covered_bytes: u64 = 0; + var gathered = false; + for (spec.parts, destination_parts) |part, *slot| { + slot.* = .{ + .source = part.source, + .source_slot = try self.sourceSlot(part.source.file_uri), + .destination = destination, + .lattice = part.lattice, + }; + covered_bytes += part.lattice.width; + gathered = gathered or part.lattice.dims.len != 0; + const strided = self.stridedRead(part); + std.debug.assert(strided.read <= strided.limit); // backend.Backend.stridedRead + } + std.debug.assert(!gathered or !spec.memory.isHost()); // backend.LoadSpec.memory + destination.* = .{ + .shape = spec.shape, + .grid = grid, + .memory = spec.memory, + .output = spec.output, + .parts = destination_parts, + .gathered = gathered, + // A gathered destination's jobs land every byte, zeros too. + .unread_bytes = .init(if (gathered) grid.total else covered_bytes), + .progress = progress, + }; + } + batch.destinations = destinations; + batch.parts = parts; + } + + /// Releases the destinations' device state, then the batch's arena. + fn destroyBatch(self: *Loader, batch: *Batch) void { + for (batch.destinations) |*destination| destination.deinit(self.allocator, self.platform.pjrt_api); + batch.destroy(); + } + + fn sourceSlot(self: *Loader, uri: []const u8) !*SourceSlot { + if (self.source_slots.get(uri)) |slot| return slot; + const owned = try self.allocator.dupe(u8, uri); + errdefer self.allocator.free(owned); + const slot = try self.allocator.create(SourceSlot); + errdefer self.allocator.destroy(slot); + slot.* = .{ .uri = owned }; + try self.source_slots.putNoClobber(self.allocator, owned, slot); + return slot; + } + + fn workerMain(self: *Loader) void { + var scratch = ReadRequest.Scratch.init(self.allocator, self.credits.blocks_per_slot, self.plan_config.block_size) catch |err| { + self.recordError(err, "allocate source worker scratch: blocks={d}", .{self.credits.blocks_per_slot}); + return; + }; + defer scratch.deinit(); + while (self.scheduler.waitForWork(self.io)) { + const request = self.claimRequest() orelse continue; + // The scheduling sentinel keeps the batch, and with it the + // request's plan, alive through `run` and error reporting. + defer request.release(); + request.run(&scratch) catch |err| { + self.recordError(err, "source request: file={s}, job={t}", .{ request.plan.source_slot.uri, request.plan.jobs[request.index].kind }); + }; + } + } + + /// Takes a credit, then the next job (`Credits`): a claimed job never + /// waits for its slot. Null, with the credit given back, when nothing + /// was left to claim. + fn claimRequest(self: *Loader) ?*ReadRequest { + const credit_wait_started = awakeNs(self.io); + const slot = self.credits.acquire(self.io); + const credit_wait_ns = awakeNs(self.io) -| credit_wait_started; + const claim = self.scheduler.claim(self.io) orelse { + self.credits.release(self.io, slot); + return null; + }; + _ = self.metrics.credit_wait_ns.fetchAdd(credit_wait_ns, .monotonic); + return ReadRequest.init(self, claim, slot); + } + + /// Stops the workers: every batch was awaited, so nothing is queued and + /// no request holds a credit or permit, and `Scheduler.fail` only wakes + /// the workers waiting for work. A worker may still hold the credit it + /// took for a claim that finds nothing (`claimRequest`); it gives it + /// back before it sees `stopping`, so every credit is free once this + /// returns. Awaiting a group that never spawned returns at once, so + /// this also cleans up a failed `create`. + fn stopWorkers(self: *Loader) void { + std.debug.assert(self.scheduler.queue.len == 0); + self.scheduler.fail(self.io); + self.worker_group.await(self.io) catch {}; + } + + /// Keeps `err` as the sticky error unless one is kept already and logs + /// it, then fails the load without waiting: unclaimed jobs retire + /// (`Scheduler.fail`) and queued pieces are released + /// (`Dma.abortReady`). Any thread may call it, a DMA callback included; + /// requests in flight stop at their next read (`readSlot`) or enqueue + /// and release their units. + /// Failing other batches than the caller's is safe from any thread: + /// `Scheduler.fail` and `Dma.abortReady` reach only live batches (their + /// docs), and `destroy` waits for the worker group before it frees what + /// the workers use. + fn recordError(self: *Loader, err: anyerror, comptime fmt: []const u8, args: anytype) void { + if (self.first_error.set(err)) { + load_log.failure(err, fmt, args); + self.scheduler.fail(self.io); + Dma.abortReady(self); + } + } + + /// Counts the plans' jobs, transfers and events only when the lines + /// would be logged. + fn logBatch(self: *Loader, batch: *const Batch, done_at: std.Io.Timestamp, successful: bool) void { + if (!std.log.logEnabled(.debug, .@"zml/io/load")) return; + const sealed_at = batch.sealed_at orelse self.created_at; + const published_at = batch.published_at orelse sealed_at; + var logical_bytes: usize = 0; + for (batch.destinations) |*destination| logical_bytes += destination.shape.byteSize(); + load_log.debug("batch completed: batch={d}, successful={}, logical_bytes={Bi:.2}, published=+{d:.3}s, sealed=+{d:.3}s, done=+{d:.3}s, elapsed={d:.3}s, source_width={d}, request_size={Bi:.2}", .{ + batch.sequence, + successful, + logical_bytes, + secondsBetween(self.created_at, published_at), + secondsBetween(self.created_at, sealed_at), + secondsBetween(self.created_at, done_at), + secondsBetween(published_at, done_at), + self.read_width, + self.plan_config.request_size, + }); + + var planning_ns: u64 = 0; + var source_bytes: u64 = 0; + var jobs: usize = 0; + var transfers: usize = 0; + var dma_submissions: usize = 0; + var kinds: std.EnumArray(std.meta.Tag(Plan.Job.Kind), usize) = .initFill(0); + var gather_jobs: usize = 0; + for (batch.plans.items) |plan| { + planning_ns += plan.planning_ns; + source_bytes += plan.source_bytes; + jobs += plan.jobs.len; + transfers += plan.transfers.len; + dma_submissions += plan.events.len; + for (plan.jobs) |job| { + kinds.getPtr(job.kind).* += 1; + // A gathered destination's landing jobs land it alone. + if (job.kind == .dma_land and plan.transfersOf(job)[0].destination.gathered) gather_jobs += 1; + } + } + const fill_jobs = kinds.get(.dma_fill) + kinds.get(.host_fill); + const host_jobs = kinds.get(.host_direct) + kinds.get(.host_bounce) + kinds.get(.host_fill); + const source_jobs = jobs - fill_jobs; + const average_read_size = if (source_jobs == 0) 0 else source_bytes / source_jobs; + const coalescing_ratio = if (source_jobs == 0) + 0 + else + @as(f64, @floatFromInt(batch.parts.len)) / + @as(f64, @floatFromInt(source_jobs)); + load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, host_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, staged_jobs={d}, gather_jobs={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ + batch.sequence, + batch.plans.items.len, + @as(f64, @floatFromInt(planning_ns)) / std.time.ns_per_s, + source_bytes, + source_jobs, + fill_jobs, + host_jobs, + batch.parts.len, + transfers, + dma_submissions, + kinds.get(.dma_land) - gather_jobs, + gather_jobs, + coalescing_ratio, + average_read_size, + }); + } + + fn logSummary(self: *Loader) void { + const reads = self.metrics.read_operations.load(.acquire); + load_log.debug("loader summary: batches={d}, successful={}, read_bytes={Bi:.2}, elapsed={d:.3}s, reads={d}, source_width={d}, request_size={Bi:.2}, credits_high_water={d} of {d}, pinned_mapped={Bi:.2}", .{ + self.batch_count, + self.first_error.get() == null, + self.metrics.read_bytes.load(.acquire), + secondsBetween(self.created_at, .now(self.io, .awake)), + reads, + self.read_width, + self.plan_config.request_size, + self.credits.high_water, + self.credits.count(), + self.workspace.mappedBytes(), + }); + load_log.debug("loader waits: credit_wait_ms_per_read={d:.3}, read_ms_per_read={d:.3}, dma_stage_ms_per_read={d:.3}, tensor_init_ms_per_read={d:.3}, ahead_init_ms={d:.3}", .{ + millisecondsPer(self.metrics.credit_wait_ns.load(.acquire), reads), + millisecondsPer(self.metrics.read_ns.load(.acquire), reads), + millisecondsPer(self.metrics.dma_stage_ns.load(.acquire), reads), + millisecondsPer(self.metrics.tensor_init_ns.load(.acquire), reads), + millisecondsPer(self.metrics.ahead_init_ns.load(.acquire), 1), + }); + const submissions = self.metrics.dma_submissions.load(.acquire); + load_log.debug("loader DMA: dma_submissions={d}, dma_submit_us_per_piece={d:.2}, dma_piece_latency_ms={d:.3}, pump_stops_empty={d}, pump_stops_full={d}", .{ + submissions, + millisecondsPer(self.metrics.dma_submit_ns.load(.acquire), submissions) * 1000, + millisecondsPer(self.metrics.dma_piece_ns.load(.acquire), submissions), + self.metrics.pump_stops_empty.load(.acquire), + self.metrics.pump_stops_full.load(.acquire), + }); + } +}; + +/// One submission: its plans (one per source file, published as each file +/// is planned), the destinations it writes and their parts, and every +/// request and event context created while loading it, all in the batch's +/// arena until it retires. `remaining` counts completion units: one per +/// published job plus a publish sentinel held until the submission is +/// sealed, and one for the allocation task while it runs +/// (`Loader.startAllocations`), taken before the sentinel is dropped. A +/// job's unit is released exactly once, by whichever of these happens: its +/// request's last reference drops (final DMA callback, a host job's end, or +/// abandonment), or `Scheduler.fail` retires it unclaimed. The batch is +/// done when `remaining` reaches zero; the awaiting task then retires it, +/// so releasing a unit is the last permitted access to the batch. +pub const Batch = struct { + /// Holds the plans with their contexts, the destinations and the parts + /// until `destroy`; only the submitting task allocates in it. + arena: std.heap.ArenaAllocator, + io: std.Io, + /// Published plans in file order: appended by the submitting task under + /// the scheduler mutex. + plans: std.ArrayList(*Plan) = .empty, + /// Their device state is the loader's, released before `destroy`. + destinations: []Destination = &.{}, + /// The destinations' parts, each destination's consecutive and in its + /// destination order: what the planner sorts by file offset. + parts: []Part = &.{}, + remaining: std.atomic.Value(usize), + done: std.Io.Event = .unset, + /// Submission number within the loader, for log correlation. + sequence: usize, + /// The first publish; null for a submission without plans. + published_at: ?std.Io.Timestamp = null, + /// Once every plan is published, before the publish sentinel goes. + sealed_at: ?std.Io.Timestamp = null, + + /// An open batch holding its publish sentinel and nothing else. The + /// sentinel keeps the batch from completing before its last plan is + /// visible. + fn create(allocator: std.mem.Allocator, io: std.Io, sequence: usize) !*Batch { + const self = try allocator.create(Batch); + self.* = .{ + .arena = .init(allocator), + .io = io, + .remaining = .init(1), + .sequence = sequence, + }; + return self; + } + + /// Releases `count` completion units. MEMORY-ORDER RULE: this must be the + /// caller's final access to the batch and to anything it owns (requests, + /// events, destinations, parts, plans; a request's slot of blocks is the + /// loader's and went back with its credit just before). The last unit + /// sets `done`, and the awaiting task frees all of it as soon as it + /// observes the event. + fn releaseUnits(self: *Batch, count: usize) void { + if (count == 0) return; + const previous = self.remaining.fetchSub(count, .acq_rel); + std.debug.assert(previous >= count); + if (previous == count) self.done.set(self.io); + } + + /// Frees the arena: the plans with their contexts, which + /// `Dma.retireBatch` must already have retired, the destinations and + /// the parts. + fn destroy(self: *Batch) void { + const allocator = self.arena.child_allocator; + self.arena.deinit(); + allocator.destroy(self); + } +}; + +/// One output buffer and what places it. Owned by the batch. Its parts may +/// come from several files, so several plans can write it: their pieces, +/// and the zero fill, all count toward the same targets (`Target.total` is +/// the placement's bytes, whichever plan submits them), and its device +/// state is created once, by the first worker that reads or fills for it +/// or, for a host-resident one, normally by the allocation task ahead of +/// them. +const Destination = struct { + /// The caller's shape, which the output buffer keeps. + shape: Shape, + /// Where `shape`, packed and placed (`Platform.placedShape`), lands on + /// its devices: what PJRT allocates and the planner cuts. + grid: ShardGrid, + /// Where its bytes go (`TensorStore.Memory`). + memory: Memory = .default, + output: *Buffer, + /// What fills it, in ascending, disjoint destination order + /// (`backend.LoadSpec`): a slice of the batch's parts. Bytes no part + /// covers are written from the loader's zero block, by the fill job of + /// the plan holding `parts[0]`. + parts: []Part, + /// Fed by strided parts: planned whole, by logical ranges, with the + /// plan holding `parts[0]` (`Planner.JobCut.appendGatherJobs`), whose + /// jobs also write its zero fill. + gathered: bool = false, + /// Bytes not read yet: the parts' widths, or for a gathered destination + /// its whole placed size (its gather jobs land the zeros too). The read + /// that brings it to zero completes the destination's item on + /// `progress`. + unread_bytes: std.atomic.Value(u64), + progress: std.Progress.Node = .none, + state: LazyOnce(DestinationState) = .{}, + + /// How its parts are planned, a class at a time in this order + /// (`Planner.sortedPartOrder`): DMA jobs, host jobs, or gathered whole. + const Delivery = enum { dma, host, gathered }; + + fn delivery(self: *const Destination) Delivery { + if (self.gathered) return .gathered; + return if (self.memory.isHost()) .host else .dma; + } + + fn ensureState(self: *Destination, direct: *Loader) StateInitError!*DestinationState { + return self.state.ensure(direct.io, DestinationState.init, .{ direct, self }); + } + + fn deinit(self: *Destination, allocator: std.mem.Allocator, api: *const pjrt.Api) void { + if (self.state.readyValue()) |state| state.deinit(allocator, api); + } + + /// The first source's name, for logs. + fn name(self: *const Destination) []const u8 { + return self.parts[0].source.name; + } + + /// Counts `bytes` as read (`unread_bytes`); the last read completes the + /// destination's one progress item. + fn recordRead(self: *Destination, bytes: u64) void { + if (self.unread_bytes.fetchSub(bytes, .acq_rel) == bytes) self.progress.completeOne(); + } +}; + +/// One source range of a destination: what the planner sorts by file offset +/// and cuts into jobs. A plain tensor is one part covering its whole +/// destination. +const Part = struct { + source: *const safetensors.Tensor, + source_slot: *SourceSlot, + destination: *Destination, + /// The spec part's; its dimensions are borrowed until the await + /// (`backend.LoadSpec`). + lattice: backend.Lattice, + + /// The part's bytes in its file, which `backend.checkParts` bounds. + fn fileRange(self: *const Part) struct { start: u64, end: u64 } { + const start = self.source.offset + self.lattice.src_offset; + return .{ .start = start, .end = start + self.lattice.width }; + } +}; + +/// One source file: every job of a plan reads the file of its slot, which +/// the first read opens, or the planner when it asks the VFS whether the +/// file is read directly (`Planner.Reads`). +const SourceSlot = struct { + uri: []const u8, + file: LazyOnce(std.Io.File) = .{}, + /// The file's read alignment once the planner decided it + /// (`Planner.Reads`): 0 when it is read exactly. + alignment: ?usize = null, + + fn ensure(self: *SourceSlot, io: std.Io) std.Io.File.OpenError!std.Io.File { + const file = try self.file.ensure(io, std.Io.Dir.openFile, .{ .cwd(), io, self.uri, .{ .mode = .read_only } }); + return file.*; + } + + fn deinit(self: *SourceSlot, io: std.Io) void { + if (self.file.readyValue()) |file| file.close(io); + } +}; + +/// One file's jobs in claim order with their transfer records and every +/// context the jobs can need, allocated by the planner: one request per +/// job and one event per planned DMA submission, handed out in +/// submission order; a host job needs only its request. Contexts hold the +/// plan's address, which the batch's arena keeps until it retires. +const Plan = struct { + /// A claimable job: its slice of the plan's transfers and what its + /// worker does for them. Its request slot is its index. + const Job = struct { + /// Where the job reads first: its read's start, a landing job's + /// first extent's; zero for a job that reads nothing. + /// `Planner.preparePlan` sorts the other jobs by it and appends + /// the fill jobs after them. + file_offset: u64, + transfer_start: usize, + transfer_len: usize, + kind: Kind, + + /// How the job's bytes reach their destinations. A destination + /// in a host memory space (`Memory.isHost`) is written + /// by the worker that reads it: going through pinned blocks and + /// a host-to-host PJRT transfer copied every byte twice and held + /// the DMA pumps, which on four B70 made Llama-3.1-8B take 2.0 s + /// into pinned_host against 0.67 s into device memory. + const Kind = union(enum) { + /// Read into pinned blocks in file order, where its transfers + /// address it, then copied to the devices by DMA. + dma_read: Read, + /// Reads its extents, whose items land their bytes in its + /// blocks: a read of its plan's file grouped by destination + /// box (`Planner.JobCut.stage`), or a gathered destination's + /// range (`Planner.JobCut.appendGatherJobs`). + dma_land: Land, + /// Reads nothing: every transfer goes from the loader's zero + /// block (`Planner.JobCut.appendFillTransfers`). + dma_fill, + /// Read straight into host-resident destinations: the read's + /// iovecs are the destination ranges, so the read is the only + /// copy. Needs a read without alignment padding whose + /// transfers address each of its bytes once, in order + /// (`Planner.JobCut.coversReadInOrder`). + host_direct: Read, + /// Read into the request's slot of blocks, then copied into + /// host-resident destinations by the worker: reads widened + /// for direct I/O, whose buffers must be aligned, and + /// overlapping parts. + host_bounce: Read, + /// Writes zeros in place, reading nothing. + host_fill, + }; + + /// `len` bytes from the job's `file_offset`, of which the first + /// `minimum_len` hold tensor data and must exist. The rest is + /// alignment padding the end of the file may cut. + const Read = struct { + len: usize, + minimum_len: usize, + }; + + /// A landing job: `len` bytes in its blocks, landed by the + /// `extent_len` extents of the plan's `extents` from + /// `extent_start`, in order. A gathered range no part covers + /// has none. + const Land = struct { + len: usize, + extent_start: usize, + extent_len: usize, + /// Its blocks start zeroed: bytes no part covers. + zeroed: bool = false, + }; + + /// The blocks of its credit's slot the job reads into or lands + /// in; none for a direct host job or a fill job. + fn blockLen(self: Job, block_size: usize) usize { + return switch (self.kind) { + .dma_read, .host_bounce => |read| Planner.blockCount(read.len, block_size), + .dma_land => |land| Planner.blockCount(land.len, block_size), + .dma_fill, .host_direct, .host_fill => 0, + }; + } + }; + + /// Bytes an extent of a landing job places in its blocks: the runs + /// `first` to `end` of `lattice`, its source offsets in the file + /// and its destination offsets logical, clipped to the logical + /// range `[clip_start, clip_end)`. A byte at logical offset `l`, + /// which `grid.locate(l)` puts at offset `o` of box `b` of the + /// destination's grid, lands at `landings[landing_start + b] +% o` + /// of the job's blocks taken end to end (`walkItem`). + const Item = struct { + destination: *Destination, + lattice: backend.Lattice, + first: u64 = 0, + end: u64 = 1, + clip_start: u64, + clip_end: u64, + landing_start: usize, + }; + + /// One read of a landing job: `len` bytes from `file_offset` of + /// `source_slot`'s file, of which the first `minimum_len` exist, + /// landed by the `item_len` items of the plan's `items` from + /// `item_start`. + const Extent = struct { + source_slot: *SourceSlot, + file_offset: u64, + len: usize, + minimum_len: usize, + /// Read straight into the job's blocks: an exact read whose + /// items' runs tile it, in order. Otherwise read into the + /// worker's bounce buffer and copied from there. + scatter: bool, + item_start: usize, + item_len: usize, + }; + + /// `len` bytes at `landing` of the job's blocks taken end to end, + /// for every device of `writer_mask` at `destination_offset` of its + /// shard. Every transfer stays inside one block, which the worker or + /// `Dma.enqueue` finds from `landing` (`blockPiece`); a direct host + /// job reads at the same landings without blocks. A fill one lands + /// at 0: a DMA one reads the zero block, a host one writes zeros. + const Transfer = struct { + destination: *Destination, + landing: usize, + writer_mask: u64, + destination_offset: usize, + len: usize, + }; + + /// The submission this plan belongs to; set when it is published, + /// under the scheduler mutex, with its completion units. + batch: *Batch = undefined, + /// The file every job of the plan reads. + source_slot: *SourceSlot, + jobs: []Job, + transfers: []Transfer, + /// The landing jobs' extents, their items and the items' landing + /// tables (one entry per box of the item's destination). + extents: []Extent = &.{}, + items: []Item = &.{}, + landings: []u64 = &.{}, + requests: []ReadRequest, + events: []Dma.EventContext, + /// Event slots handed out so far, one `fetchAdd` per submission from + /// any device's pump. + events_used: std.atomic.Value(usize) = .init(0), + /// The tensor bytes the jobs read. + source_bytes: u64, + /// How long the plan took to cut. + planning_ns: u64 = 0, + /// Next job to claim; owned by the scheduler mutex. + cursor: usize = 0, + + fn transfersOf(self: *const Plan, job: Job) []const Transfer { + return self.transfers[job.transfer_start..][0..job.transfer_len]; + } +}; + +/// Builds immutable per-file source jobs and transfer records. Their claim +/// order (file order) is decided here once. +/// For DeepSeek-V4-Flash (148.65 GiB, 69,187 tensors), tensor-local reads made +/// about 69,445 source calls despite a 16 MiB request limit. Coalescing reduced +/// that to 9,524. A rigid grid still produced 78,665 DMA pieces; tensor-safe +/// cuts kept the same read count with 69,572 pieces. Per-tensor device buffers +/// remain intentional: reducing submissions below roughly tensors x destinations would require +/// packed device allocations or device-side scattering, not another read cut. +const Planner = struct { + /// What every plan of a loader is cut by. + const Config = struct { + block_size: usize, + /// The longest read, a whole number of blocks + /// (`backend.requestSize`). + request_size: usize, + /// The one-read limit of a strided part (`stridedRead`, which caps + /// it at `request_size`): the request of the smallest block + /// calibration may pick, so a refusal does not follow its pick. + gather_limit: usize = std.math.maxInt(usize), + /// Submissions a DMA job must save to be staged (`JobCut.stage`): + /// one device's in-flight pieces. + minimum_staging_gain: usize = max_dma_pieces_per_device, + }; + + /// How each file is read: aligned when it is local and the VFS that + /// opens it reads it directly under `policy`, or as `predict` answered + /// for it, decided once per file and kept in its slot. A remote file is + /// read exact without being opened here. Without a VFS (`io` is none, or + /// no alignment) every file takes `alignment`. A gather job reads files + /// other than its plan's, so it asks here for each. `predict` tells, + /// before any file is planned, how the checkpoint's files will read. + /// Widening buffered reads is not free: a widened read served from the + /// page cache costs the DMA sources their block alignment for nothing, + /// and warm replicated Llama on four GB300 lost 3-6% when its first + /// tensor no longer landed at block offset zero. So a file asks the VFS + /// actually owning its handle (`VFS.fromIo`, in `init`), both to avoid + /// that cost and to avoid indexing another VFS's handle table; an `io` + /// that is no VFS never reads directly. + const Reads = struct { + io: std.Io, + vfs: ?*VFS, + policy: VFS.DirectIo, + /// Every read of a file read directly starts and ends at a multiple + /// of this; 0 reads every file exact. + alignment: usize, + /// The files `predict` answered for, by real path as the registry + /// names them (`safetensors.Tensor.file_uri`), each with the policy + /// that decides it as answered: under `auto`, `on` for a file that + /// reads direct and `off` for one that reads buffered. Under `on`, + /// which samples nothing, every file keeps `on`, and a filesystem + /// refusing the flag is logged when the file is planned. Owned until + /// `deinit`. + predicted: std.StringHashMapUnmanaged(VFS.DirectIo) = .empty, + + fn init(io: std.Io, policy: VFS.DirectIo) Reads { + const alignment = backend.readAlignment(io, policy); + return .{ .io = io, .vfs = if (alignment != 0) VFS.fromIo(io) else null, .policy = policy, .alignment = alignment }; + } + + fn deinit(self: *Reads, allocator: std.mem.Allocator) void { + var paths = self.predicted.keyIterator(); + while (paths.next()) |path| allocator.free(path.*); + self.predicted.deinit(allocator); + } + + /// The file's alignment, decided the first time it is asked: a + /// local file is opened here, and decided as `predict` answered for + /// it, else under `policy`. One that fails to open is read exact: + /// its reads fail with the same error (`LazyOnce`), which `readSlot` + /// logs with the path. + fn alignmentOf(self: Reads, slot: *SourceSlot) error{Canceled}!usize { + const vfs = self.vfs orelse return self.alignment; + if (slot.alignment) |known| return known; + // A predicted file is decided as predicted, not sampled again: + // the prediction's sample queued the read of each page it missed + // and of the kernel's readahead, and a second sample of those + // pages once the reads have landed finds the file cached + // (`VFS.predictDirectIo`). Under `on`, a filesystem refusing the + // flag still reads it buffered. + const policy = self.predicted.get(slot.uri) orelse self.policy; + const direct = vfs.isLocal(slot.uri) and vfs.useDirectIo(slot.ensure(self.io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return 0, + }, policy); + const alignment = if (direct) self.alignment else 0; + slot.alignment = alignment; + return alignment; + } + + /// The widest alignment `alignmentOf` can give the file, without + /// opening it: 0 for a remote file. + fn boundOf(self: Reads, uri: []const u8) usize { + const vfs = self.vfs orelse return self.alignment; + return if (vfs.isLocal(uri)) self.alignment else 0; + } + + /// How the checkpoint's local safetensors files will read, asked + /// once, before any is planned, to place the arenas: each as + /// `VFS.predictDirectIo` answers under `policy`, an answer kept in + /// `predicted` for `alignmentOf` to decide the file by. The files + /// are those `TensorRegistry.fromPath` reads for the checkpoint + /// (`safetensors.checkpointFiles`, which reads its index). Nothing + /// without a VFS, for a checkpoint that is not local, or for files + /// that do not resolve or open: the registry reports those. Keeps + /// no file open, and logs nothing. A wrong prediction (residency + /// changing before the file is planned) costs speed, never bytes. + fn predict(self: *Reads, allocator: std.mem.Allocator, checkpoint: ?[]const u8) error{ Canceled, OutOfMemory }!Prediction { + std.debug.assert(self.predicted.count() == 0); + const vfs = self.vfs orelse return .{}; + const path = checkpoint orelse return .{}; + if (!vfs.isLocal(path)) return .{}; + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const dir, const names = safetensors.checkpointFiles(arena.allocator(), self.io, path) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.OutOfMemory => return error.OutOfMemory, + else => return .{}, + }; + defer dir.close(self.io); + var prediction: Prediction = .{}; + var real_path_buffer: [std.fs.max_path_bytes]u8 = undefined; + for (names) |name| { + const real_path_len = dir.realPathFile(self.io, name, &real_path_buffer) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => continue, + }; + const real_path = real_path_buffer[0..real_path_len]; + // A symlink and the file it leads to are sampled once. + if (self.predicted.contains(real_path)) continue; + const file = try vfs.predictDirectIo(dir, name, self.policy) orelse continue; + const owned = try allocator.dupe(u8, real_path); + errdefer allocator.free(owned); + try self.predicted.putNoClobber(allocator, owned, if (file.direct or self.policy == .on) .on else .off); + prediction.add(file); + } + return prediction; + } + + /// The bytes of a checkpoint's local safetensors files, and of those + /// that will read direct (`predict`). + const Prediction = struct { + files: usize = 0, + bytes: u64 = 0, + direct_bytes: u64 = 0, + + fn add(self: *Prediction, file: VFS.DirectIoPrediction) void { + self.files += 1; + self.bytes += file.size; + if (file.direct) self.direct_bytes += file.size; + } + + /// Whether the arenas are placed for direct reads + /// (`host_memory.Workspace.init`): files holding at least four + /// fifths of the bytes read direct. In measurements on two GB300 + /// of one Grace node loading Qwen3.8-27B-FP8 (28.75 GiB), + /// spreading the arenas over both nodes saved direct reads about + /// 4.2 ms per GiB forced direct and 4.0 cold under auto, and cost + /// reads through the page cache 10 to 16 ms per GiB: it pays from + /// about 0.71 to 0.80 of the bytes read direct, and four fifths, + /// at the top of that range, leans to the devices' nodes, whose + /// mistake costs less per GiB. + fn direct(self: Prediction) bool { + return self.bytes != 0 and self.direct_bytes * 5 >= self.bytes * 4; + } + }; + }; + + /// Plans `parts` one file at a time and publishes each plan as soon as + /// it exists, so workers claim the first file while the rest is planned, + /// each plan in the batch's arena. Stops at the first error; the caller + /// seals or fails the batch. `reads` says how each file is read + /// (`Reads`). + /// Whole-model planning delayed the first read by 0.32 s for DeepSeek's + /// 46 files on one MI300X; publishing per file overlaps the rest with IO. + fn publishFiles( + scheduler: *Scheduler, + io: std.Io, + batch: *Batch, + parts: []Part, + config: Config, + reads: Reads, + ) !void { + const order = try sortedPartOrder(scheduler.allocator, parts); + defer scheduler.allocator.free(order); + var file_start: usize = 0; + while (file_start < order.len) { + const file_end = fileGroupEnd(parts, order, file_start); + const planning_started: std.Io.Timestamp = .now(io, .awake); + const plan = try preparePlan(scheduler.allocator, batch.arena.allocator(), parts, order[file_start..file_end], config, reads); + plan.planning_ns = @intCast(@max(planning_started.untilNow(io, .awake).nanoseconds, 0)); + try scheduler.publish(io, batch, plan); + if (batch.published_at == null) batch.published_at = .now(io, .awake); + file_start = file_end; + } + } + + /// Plans one file: `order` indexes its `parts` by delivery class, then + /// offset and size (`sortedPartOrder`). `JobCut.cutRuns` cuts the DMA + /// and the host parts at the file's alignment (`Reads.alignmentOf`), + /// `JobCut.appendGatherJobs` each gathered destination with the file of + /// its first part; the jobs merge in file order, then + /// `JobCut.appendFillJob` appends each class's zero fill. The plan goes + /// in `arena` (`JobCut.finish`), the cut in `allocator`. + fn preparePlan( + allocator: std.mem.Allocator, + arena: std.mem.Allocator, + parts: []Part, + order: []const usize, + config: Config, + reads: Reads, + ) !*Plan { + const source_slot = parts[order[0]].source_slot; + for (order) |part_index| std.debug.assert(std.mem.eql(u8, parts[part_index].source.file_uri, parts[order[0]].source.file_uri)); + const dma_end = deliveryEnd(parts, order, 0, .dma); + const host_end = deliveryEnd(parts, order, dma_end, .host); + const alignment = try reads.alignmentOf(source_slot); + var cut: JobCut = .{ + .allocator = allocator, + .parts = parts, + .source_slot = source_slot, + .config = config, + .reads = reads, + .alignment = alignment, + .maximum_job_len = maximumJobLen(config.request_size, alignment), + // A gather job's extents may be in files read directly while + // this one is not, so its ranges leave room for the widest + // alignment. + .gather_len = maximumJobLen(config.request_size, reads.alignment), + }; + defer cut.deinit(); + try cut.cutRuns(order[0..dma_end], false); + try cut.cutRuns(order[dma_end..host_end], true); + for (order[host_end..]) |part_index| { + const part = &parts[part_index]; + std.debug.assert(part.destination.gathered); + if (part == &part.destination.parts[0]) try cut.appendGatherJobs(part.destination); + } + // Jobs are claimed in planning order, which is file order (a + // gather job by its first read). A byte-fair order across + // destination devices was measured against it on the two hosts + // where the DMA pump is not the ceiling and made no difference. + // Each class is in file order already; the stable sort interleaves + // them. + std.mem.sort(Plan.Job, cut.jobs.items, {}, struct { + fn lessThan(_: void, lhs: Plan.Job, rhs: Plan.Job) bool { + return lhs.file_offset < rhs.file_offset; + } + }.lessThan); + + // The zero fill of each destination goes out once, with the plan + // of its first part; it depends on no read, so one job after this + // file's reads carries all of it, one per delivery class. + try cut.appendFillJob(order[0..dma_end], false); + try cut.appendFillJob(order[dma_end..host_end], true); + return cut.finish(arena); + } + + /// The end of the parts of `delivery` in `order` from `start` on. + fn deliveryEnd(parts: []const Part, order: []const usize, start: usize, delivery: Destination.Delivery) usize { + var end = start; + while (end < order.len and parts[order[end]].destination.delivery() == delivery) : (end += 1) {} + return end; + } + + /// The jobs and transfers of one plan as `preparePlan` cuts them. + const JobCut = struct { + /// One part's bytes in a read job: the logical range `[start, end)` + /// of its destination, from `file_offset` of the file on. + const Segment = struct { destination: *Destination, start: u64, end: u64, file_offset: u64 }; + /// A segment's bytes on box `box` of its destination: `len` from + /// `box_offset`, and where a staged job lands them. + const Entry = struct { segment: u32, box: u32, group: u32 = 0, box_offset: u64, len: u64, landing: u64 = 0 }; + /// What a staged job lands together. + const Group = struct { destination: *Destination, box: u32 }; + + allocator: std.mem.Allocator, + parts: []Part, + /// The file every job of the plan reads. + source_slot: *SourceSlot, + config: Config, + reads: Reads, + /// The alignment of the plan's file (`Reads.alignmentOf`). + alignment: usize, + maximum_job_len: usize, + /// The longest logical range of a gather job. + gather_len: usize, + jobs: std.ArrayList(Plan.Job) = .empty, + transfers: std.ArrayList(Plan.Transfer) = .empty, + items: std.ArrayList(Plan.Item) = .empty, + landings: std.ArrayList(u64) = .empty, + extents: std.ArrayList(Plan.Extent) = .empty, + safe_boundaries: std.ArrayList(u64) = .empty, + /// A read job's segments before they are placed, and the staging + /// scratch. + segments: std.ArrayList(Segment) = .empty, + entries: std.ArrayList(Entry) = .empty, + groups: std.AutoHashMapUnmanaged(Group, u32) = .empty, + source_bytes: u64 = 0, + + /// The plan of the jobs cut, copied into `arena` with every context + /// its jobs can need, allocated before publication makes it visible + /// to workers: a request per job and an event per writer of a DMA + /// job's transfer. + fn finish(self: *JobCut, arena: std.mem.Allocator) !*Plan { + const jobs = self.jobs.items; + const transfers = self.transfers.items; + var dma_submissions: usize = 0; + for (jobs) |job| switch (job.kind) { + .dma_read, .dma_land, .dma_fill => dma_submissions += submissionCount(transfers[job.transfer_start..][0..job.transfer_len]), + .host_direct, .host_bounce, .host_fill => {}, + }; + const requests = try arena.alloc(ReadRequest, jobs.len); + @memset(requests, ReadRequest.idle); + const plan = try arena.create(Plan); + plan.* = .{ + .source_slot = self.source_slot, + .jobs = try arena.dupe(Plan.Job, jobs), + .transfers = try arena.dupe(Plan.Transfer, transfers), + .extents = try arena.dupe(Plan.Extent, self.extents.items), + .items = try arena.dupe(Plan.Item, self.items.items), + .landings = try arena.dupe(u64, self.landings.items), + .requests = requests, + .events = try arena.alloc(Dma.EventContext, dma_submissions), + .source_bytes = self.source_bytes, + }; + return plan; + } + + fn deinit(self: *JobCut) void { + const allocator = self.allocator; + self.groups.deinit(allocator); + self.entries.deinit(allocator); + self.segments.deinit(allocator); + self.safe_boundaries.deinit(allocator); + self.extents.deinit(allocator); + self.landings.deinit(allocator); + self.items.deinit(allocator); + self.transfers.deinit(allocator); + self.jobs.deinit(allocator); + } + + /// Cuts the runs of `order` (part indices of one delivery class in + /// file order) into jobs: each run of touching or overlapping + /// ranges into the minimum number of jobs, cut at part-safe + /// boundaries where that number allows. No job depends on another, + /// since every DMA piece is submitted as soon as its block is read. + /// With an alignment (`Reads.alignmentOf`), a job's read widens to + /// aligned bounds around its range (`maximumJobLen` leaves room for + /// it), and the transfers address the widened read. A DMA job's + /// segments are placed in its blocks by `placeDma`. A host job is + /// `host_direct` only when its read is exact and its transfers + /// cover it in order (`coversReadInOrder`), otherwise `host_bounce`. + fn cutRuns(self: *JobCut, order: []const usize, host: bool) !void { + const allocator = self.allocator; + const parts = self.parts; + const alignment = self.alignment; + const maximum_job_len = self.maximum_job_len; + var run_cursor: usize = 0; + while (run_cursor < order.len) { + self.safe_boundaries.clearRetainingCapacity(); + const first_range = parts[order[run_cursor]].fileRange(); + std.debug.assert(first_range.end > first_range.start); // backend.LoadSpec: parts are non-empty + const first_offset = first_range.start; + var run_end = first_range.end; + var run_part_end = run_cursor + 1; + while (run_part_end < order.len) : (run_part_end += 1) { + const candidate = parts[order[run_part_end]].fileRange(); + if (candidate.start > run_end) break; + // A touching range starts where every preceding range has + // ended. Unlike an arbitrary part end, this is safe even + // when the batch contains overlapping or duplicate ranges. + if (candidate.start == run_end and + (self.safe_boundaries.items.len == 0 or + self.safe_boundaries.items[self.safe_boundaries.items.len - 1] != candidate.start)) + { + try self.safe_boundaries.append(allocator, candidate.start); + } + run_end = @max(run_end, candidate.end); + } + + var job_start = first_offset; + var candidate_start = run_cursor; + var boundary_cursor: usize = 0; + const run_len = run_end - first_offset; + const maximum_job_len_u64: u64 = @intCast(maximum_job_len); + var jobs_remaining: usize = @intCast(run_len / maximum_job_len_u64 + + @intFromBool(run_len % maximum_job_len_u64 != 0)); + while (jobs_remaining != 0) { + const safe_boundaries = self.safe_boundaries.items; + const hard_end = @min( + run_end, + std.math.add(u64, job_start, maximum_job_len) catch run_end, + ); + const job_end = if (jobs_remaining == 1) + run_end + else boundary: { + // Less than the run's length, which is a u64. + const remaining_capacity = @as(u64, @intCast(jobs_remaining - 1)) * maximum_job_len_u64; + const minimum_end = @max(job_start + 1, run_end - remaining_capacity); + const maximum_end = @min( + hard_end, + run_end - @as(u64, @intCast(jobs_remaining - 1)), + ); + while (boundary_cursor < safe_boundaries.len and + safe_boundaries[boundary_cursor] <= job_start) + { + boundary_cursor += 1; + } + var scan = boundary_cursor; + var selected: ?u64 = null; + while (scan < safe_boundaries.len and + safe_boundaries[scan] <= maximum_end) : (scan += 1) + { + if (safe_boundaries[scan] >= minimum_end) + selected = safe_boundaries[scan]; + } + boundary_cursor = scan; + break :boundary selected orelse maximum_end; + }; + const job_len: usize = @intCast(job_end - job_start); + const read_start = if (alignment == 0) job_start else std.mem.alignBackward(u64, job_start, alignment); + const read_end = if (alignment == 0) job_end else std.mem.alignForward(u64, job_end, alignment); + const read_len: usize = @intCast(read_end - read_start); + const transfer_start = self.transfers.items.len; + // The job's segments first: where they land in its + // blocks depends on all of them. + self.segments.clearRetainingCapacity(); + while (candidate_start < run_part_end) { + const candidate = parts[order[candidate_start]].fileRange(); + if (candidate.end > job_start) break; + candidate_start += 1; + } + for (order[candidate_start..run_part_end]) |part_index| { + const part = &parts[part_index]; + const range = part.fileRange(); + if (range.start >= job_end) break; + const intersection_start = @max(job_start, range.start); + const intersection_end = @min(job_end, range.end); + if (intersection_start >= intersection_end) continue; + const logical_start = part.lattice.dst_offset + (intersection_start - range.start); + try self.segments.append(allocator, .{ + .destination = part.destination, + .start = logical_start, + .end = logical_start + (intersection_end - intersection_start), + .file_offset = intersection_start, + }); + } + std.debug.assert(self.segments.items.len != 0); + const read: Plan.Job.Read = .{ .len = read_len, .minimum_len = @intCast(job_end - read_start) }; + const kind: Plan.Job.Kind = if (host) host: { + for (self.segments.items) |segment| try self.appendRunTransfers(transfer_start, segment, read_start); + break :host if (alignment == 0 and coversReadInOrder(self.transfers.items[transfer_start..], read_len)) + .{ .host_direct = read } + else + .{ .host_bounce = read }; + } else try self.placeDma(read, read_start, job_start, job_len); + try self.jobs.append(allocator, .{ + .file_offset = read_start, + .transfer_start = transfer_start, + .transfer_len = self.transfers.items.len - transfer_start, + .kind = kind, + }); + self.source_bytes +|= @intCast(job_len); + job_start = job_end; + jobs_remaining -= 1; + } + std.debug.assert(job_start == run_end); + run_cursor = run_part_end; + } + } + + /// Places a DMA job's segments (`segments`, in file order) in its + /// blocks. Blocks holding the read in file order give one transfer + /// per run (`ShardGrid.run`): a tensor sharded along any axis but + /// its first takes one per row and device, as a column-sharded + /// `o_proj` or `down_proj` does. A staged job lands its read grouped + /// by destination box instead (`stage`), so a box takes one piece + /// per block. Llama-3.1-8B sharded like that on four B70, page + /// cache warm, planned 1.05M submissions and loaded in 5.1 s (~1.2 + /// ms per piece, 64 in flight per device); staged, 5,468 and 0.71 + /// s, against 0.67 s sharded by rows. Staging costs a read + /// scattered over the runs, or for a direct read a copy through the + /// worker's bounce buffer (`landJob`), so a job is staged only where + /// it saves `Config.minimum_staging_gain` submissions, and only when + /// its segments tile its bytes once: overlapping parts send a source + /// byte to two segments, and one read cannot land it twice. + fn placeDma(self: *JobCut, read: Plan.Job.Read, read_start: u64, data_start: u64, job_len: usize) !Plan.Job.Kind { + const block_size = self.config.block_size; + const transfer_start = self.transfers.items.len; + // File order takes a submission per writer of every run, cut by + // blocks, less where runs of adjacent segments continue each other + // or a run ends where a block does: at most `slack` fewer. + var in_file_order: usize = 0; + var slack: usize = 0; + var tiles = true; + var next = data_start; + for (self.segments.items) |segment| { + if (segment.file_offset != next) tiles = false; + next = segment.file_offset + (segment.end - segment.start); + const grid = &segment.destination.grid; + const landing = segment.file_offset - read_start; + const crossings = (landing + segment.end - segment.start - 1) / block_size - landing / block_size; + in_file_order += (grid.runCount(segment.start, segment.end) + crossings) * grid.replicas(); + slack += (1 + crossings) * grid.replicas(); + } + if (tiles and next == data_start + job_len) { + if (try self.stage(read, read_start, job_len, in_file_order, slack)) |land| return .{ .dma_land = land }; + } + for (self.segments.items) |segment| try self.appendRunTransfers(transfer_start, segment, read_start); + return .{ .dma_read = read }; + } + + /// Lands the job's segments box by box: their bytes on each box of a + /// destination, grouped by (destination, box) in order of first + /// appearance, each group in box order. The job's read is its one + /// extent, and one `Item` per segment tells the worker where its bytes + /// go (`landJob`). Null, with nothing appended, when that saves + /// fewer than `Config.minimum_staging_gain` submissions against file + /// order, estimated at `in_file_order` and at most `slack` fewer: + /// counted exactly, by listing the runs, only when that decides. + fn stage(self: *JobCut, read: Plan.Job.Read, read_start: u64, job_len: usize, in_file_order: usize, slack: usize) !?Plan.Job.Land { + const allocator = self.allocator; + const segments = self.segments.items; + self.entries.clearRetainingCapacity(); + self.groups.clearRetainingCapacity(); + for (segments, 0..) |segment, index| { + const grid = &segment.destination.grid; + for (0..grid.box_count) |box| { + const low = grid.rankIn(box, segment.start); + const high = grid.rankIn(box, segment.end); + if (high == low) continue; + const group = try self.groups.getOrPut(allocator, .{ .destination = segment.destination, .box = @intCast(box) }); + if (!group.found_existing) group.value_ptr.* = @intCast(self.groups.count() - 1); + try self.entries.append(allocator, .{ .segment = @intCast(index), .box = @intCast(box), .group = group.value_ptr.*, .box_offset = low, .len = high - low }); + } + } + const entries = self.entries.items; + std.mem.sort(Entry, entries, {}, struct { + fn lessThan(_: void, lhs: Entry, rhs: Entry) bool { + if (lhs.group != rhs.group) return lhs.group < rhs.group; + if (lhs.box_offset != rhs.box_offset) return lhs.box_offset < rhs.box_offset; + return lhs.segment < rhs.segment; + } + }.lessThan); + // The read's own order when every box of a segment is one run and + // the groups come in the order of the runs. + var runs: u64 = 0; + for (segments) |segment| runs += segment.destination.grid.runCount(segment.start, segment.end); + if (runs == entries.len) { + const in_order = for (entries[1..], entries[0 .. entries.len - 1]) |entry, previous| { + if (entry.segment < previous.segment or (entry.segment == previous.segment and entry.box < previous.box)) break false; + } else true; + if (in_order) return null; + } + const staged_start = self.transfers.items.len; + var landing: u64 = 0; + for (entries) |*entry| { + const destination = segments[entry.segment].destination; + entry.landing = landing; + try self.appendBlocked(staged_start, .{ + .destination = destination, + .landing = @intCast(landing), + .writer_mask = destination.grid.writers[entry.box], + .destination_offset = @intCast(entry.box_offset), + .len = @intCast(entry.len), + }); + landing += entry.len; + } + std.debug.assert(landing == job_len); + const needed = submissionCount(self.transfers.items[staged_start..]) +| self.config.minimum_staging_gain; + var file_order_wins = needed > in_file_order; + if (!file_order_wins and needed +| slack > in_file_order) { + const exact_start = self.transfers.items.len; + for (segments) |segment| try self.appendRunTransfers(exact_start, segment, read_start); + file_order_wins = needed > submissionCount(self.transfers.items[exact_start..]); + self.transfers.shrinkRetainingCapacity(exact_start); + } + if (file_order_wins) { + self.transfers.shrinkRetainingCapacity(staged_start); + return null; + } + + // A landing table per segment, one entry per box of its + // destination. + const item_start = self.items.items.len; + for (segments) |segment| { + const landing_start = self.landings.items.len; + try self.landings.appendNTimes(allocator, 0, segment.destination.grid.box_count); + try self.items.append(allocator, .{ + .destination = segment.destination, + .lattice = .{ .src_offset = segment.file_offset, .dst_offset = segment.start, .width = segment.end - segment.start }, + .clip_start = segment.start, + .clip_end = segment.end, + .landing_start = landing_start, + }); + } + for (entries) |entry| { + const item = self.items.items[item_start + entry.segment]; + self.landings.items[item.landing_start + entry.box] = entry.landing -% entry.box_offset; + } + // Only a direct read has padding around its tensor bytes, which + // goes through the worker's bounce buffer. + const extent_start = self.extents.items.len; + try self.extents.append(allocator, .{ + .source_slot = self.source_slot, + .file_offset = read_start, + .len = read.len, + .minimum_len = read.minimum_len, + .scatter = self.alignment == 0, + .item_start = item_start, + .item_len = segments.len, + }); + return .{ .len = job_len, .extent_start = extent_start, .extent_len = 1 }; + } + + /// Appends `segment`'s transfers with its bytes where the read holds + /// them, one per run (`ShardGrid.run`) and block, merged with the + /// previous transfer from `transfer_start` on where block, writers + /// and both offsets continue it: parts adjacent in the file and in + /// the destination go out as one transfer. + fn appendRunTransfers(self: *JobCut, transfer_start: usize, segment: Segment, read_start: u64) !void { + const destination = segment.destination; + const grid = &destination.grid; + var at = segment.start; + while (at < segment.end) { + const stop = grid.runEnd(at, segment.end); + const box, const box_offset = grid.locate(at); + try self.appendBlocked(transfer_start, .{ + .destination = destination, + .landing = @intCast(segment.file_offset + (at - segment.start) - read_start), + .writer_mask = grid.writers[box], + .destination_offset = @intCast(box_offset), + .len = @intCast(stop - at), + }); + at = stop; + } + } + + /// Plans a gathered destination (`Destination.gathered`) whole: its + /// logical bytes cut into ranges of at most `gather_len`, each one + /// job. For every part with runs in the range the job reads the + /// source range those runs come from (an `Extent`, from whichever + /// file holds it), and it lands the range in its blocks box by box, + /// one transfer per box and block, its bytes no part covers zero. + /// A part whose source runs are out of order (a broadcast repeats + /// them, a transpose permutes them) reads its whole source range + /// in every job (`stridedRead`). An interleave of members reads + /// each member's rows of the range, so the range lands in one piece + /// per box and block whatever the interleave's granularity. + fn appendGatherJobs(self: *JobCut, destination: *Destination) !void { + const allocator = self.allocator; + const grid = &destination.grid; + const parts = destination.parts; + // reach[i]: the furthest destination byte parts[0..=i] reach, so + // a range's first candidate is found by bisection. + const reach = try allocator.alloc(u64, parts.len); + defer allocator.free(reach); + var furthest: u64 = 0; + for (parts, reach) |*part, *at| { + const spans = part.lattice.spans().?; + furthest = @max(furthest, part.lattice.dst_offset + spans.dst); + at.* = furthest; + } + var start: u64 = 0; + while (start < grid.total) { + const end = @min(grid.total, start + self.gather_len); + const transfer_start = self.transfers.items.len; + const extent_start = self.extents.items.len; + const landing_start = self.landings.items.len; + try self.landings.appendNTimes(allocator, 0, grid.box_count); + var landing: u64 = 0; + for (0..grid.box_count) |box| { + const low = grid.rankIn(box, start); + const high = grid.rankIn(box, end); + if (high == low) continue; + self.landings.items[landing_start + box] = landing -% low; + try self.appendBlocked(transfer_start, .{ + .destination = destination, + .landing = @intCast(landing), + .writer_mask = grid.writers[box], + .destination_offset = @intCast(low), + .len = @intCast(high - low), + }); + landing += high - low; + } + std.debug.assert(landing == end - start); + + var covered: u64 = 0; + var index = std.sort.partitionPoint(u64, reach, start, struct { + fn below(limit: u64, at: u64) bool { + return at <= limit; + } + }.below); + while (index < parts.len and parts[index].lattice.dst_offset < end) : (index += 1) { + const part = &parts[index]; + const first = part.lattice.firstEndingAfter(start); + const last = part.lattice.firstStartingFrom(end); + if (first >= last) continue; + covered += part.lattice.coveredIn(start, end); + try self.appendExtent(part, first, last, start, end, landing_start); + } + std.debug.assert(covered <= end - start); + const job_len: usize = @intCast(end - start); + const extent_len = self.extents.items.len - extent_start; + try self.jobs.append(allocator, .{ + .file_offset = if (extent_len != 0) self.extents.items[extent_start].file_offset else 0, + .transfer_start = transfer_start, + .transfer_len = self.transfers.items.len - transfer_start, + .kind = .{ .dma_land = .{ + .len = job_len, + .extent_start = extent_start, + .extent_len = extent_len, + .zeroed = covered < end - start, + } }, + }); + start = end; + } + } + + /// The extent reading `part`'s runs `first` to `last` (exclusive) + /// clipped to `[start, end)`, with the one item that places them: + /// from the first run's first byte in the range to the last run's + /// last one when the part's source runs are in order, else the + /// part's whole source range. + fn appendExtent(self: *JobCut, part: *const Part, first: u64, last: u64, start: u64, end: u64, landing_start: usize) !void { + const lattice = part.lattice; + const width = lattice.width; + var source_start = lattice.src_offset; + var source_end = lattice.src_offset + lattice.spans().?.src; + const ordered = lattice.srcOrdered(); + if (ordered) { + const head = lattice.at(first); + const tail = lattice.at(last - 1); + source_start = head.src + (start -| head.dst); + source_end = tail.src + width - ((tail.dst + width) -| end); + } + const file_start = part.source.offset + source_start; + const file_end = part.source.offset + source_end; + const alignment = try self.reads.alignmentOf(part.source_slot); + const read_start = if (alignment == 0) file_start else std.mem.alignBackward(u64, file_start, alignment); + const read_end = if (alignment == 0) file_end else std.mem.alignForward(u64, file_end, alignment); + // The range of a part whose runs are back to back is the job's; + // any other part's whole range fits (`stridedRead`). + std.debug.assert(read_end - read_start <= self.config.request_size); + try self.extents.append(self.allocator, .{ + .source_slot = part.source_slot, + .file_offset = read_start, + .len = @intCast(read_end - read_start), + .minimum_len = @intCast(file_end - read_start), + .scatter = alignment == 0 and ordered and lattice.srcTiled(), + .item_start = self.items.items.len, + .item_len = 1, + }); + try self.items.append(self.allocator, .{ + .destination = part.destination, + .lattice = .{ .src_offset = part.source.offset + lattice.src_offset, .dst_offset = lattice.dst_offset, .width = width, .dims = lattice.dims }, + .first = first, + .end = last, + .clip_start = start, + .clip_end = end, + .landing_start = landing_start, + }); + self.source_bytes +|= file_end - file_start; + } + + /// Appends `transfer`, cut at block boundaries, merged with the + /// previous transfer from `transfer_start` on where block, writers + /// and both offsets continue it. + fn appendBlocked(self: *JobCut, transfer_start: usize, transfer: Plan.Transfer) !void { + const block_size = self.config.block_size; + var cursor: usize = 0; + while (cursor < transfer.len) { + const at = transfer.landing + cursor; + const in_block = at % block_size; + const take = @min(transfer.len - cursor, block_size - in_block); + const destination_offset = transfer.destination_offset + cursor; + cursor += take; + // A transfer that ends where this one starts is in its + // block unless a block starts here. + if (self.transfers.items.len > transfer_start and in_block != 0) { + const previous = &self.transfers.items[self.transfers.items.len - 1]; + if (previous.destination == transfer.destination and + previous.writer_mask == transfer.writer_mask and + previous.landing + previous.len == at and + previous.destination_offset + previous.len == destination_offset) + { + previous.len += take; + continue; + } + } + try self.transfers.append(self.allocator, .{ + .destination = transfer.destination, + .landing = at, + .writer_mask = transfer.writer_mask, + .destination_offset = destination_offset, + .len = take, + }); + } + } + + fn submissionCount(transfers: []const Plan.Transfer) usize { + var count: usize = 0; + for (transfers) |transfer| count += @popCount(transfer.writer_mask); + return count; + } + + /// The zero fill of the destinations whose first part is among + /// `order`, the file's parts of one delivery class, as one job. A DMA + /// fill job reads one block, the zero block; a host one writes zeros + /// in place. A gathered destination's jobs write its zeros + /// themselves. + fn appendFillJob(self: *JobCut, order: []const usize, host: bool) !void { + const fill_start = self.transfers.items.len; + for (order) |part_index| { + const part = &self.parts[part_index]; + const destination = part.destination; + if (part != &destination.parts[0]) continue; + try self.appendFillTransfers(fill_start, destination); + } + const fill_transfers = self.transfers.items[fill_start..]; + if (fill_transfers.len == 0) return; + try self.jobs.append(self.allocator, .{ + .file_offset = 0, + .transfer_start = fill_start, + .transfer_len = fill_transfers.len, + .kind = if (host) .host_fill else .dma_fill, + }); + } + + /// Appends the zero-fill transfers of `destination`: every logical + /// range no part covers (the gaps between its parts and the tail), + /// placed by its grid like read bytes, each piece at most one block + /// at landing 0, where a DMA job reads the zero block. Pieces count + /// toward their targets' totals, so the fill completes each + /// device's placement bytes alongside the parts. + fn appendFillTransfers(self: *JobCut, transfer_start: usize, destination: *Destination) !void { + var covered_end: u64 = 0; + for (destination.parts) |part| { + const lattice = part.lattice; + if (lattice.dst_offset > covered_end) try self.appendFillRange(transfer_start, destination, covered_end, lattice.dst_offset); + covered_end = lattice.dst_offset + lattice.width; + } + if (covered_end < destination.grid.total) try self.appendFillRange(transfer_start, destination, covered_end, destination.grid.total); + } + + /// The fill of the logical range `[start, end)`: one range of each box + /// it touches, cut by blocks, merged where one writer set's pieces + /// are contiguous. + fn appendFillRange(self: *JobCut, transfer_start: usize, destination: *Destination, start: u64, end: u64) !void { + const block_size = self.config.block_size; + const grid = &destination.grid; + for (0..grid.box_count) |box| { + var destination_offset: usize = @intCast(grid.rankIn(box, start)); + var remaining: usize = @intCast(grid.rankIn(box, end) - destination_offset); + const writer_mask = grid.writers[box]; + while (remaining != 0) { + if (self.transfers.items.len > transfer_start) merge: { + const previous = &self.transfers.items[self.transfers.items.len - 1]; + if (previous.destination != destination or previous.writer_mask != writer_mask or + previous.destination_offset + previous.len != destination_offset or + previous.len == block_size) + break :merge; + const grown = @min(remaining, block_size - previous.len); + previous.len += grown; + destination_offset += grown; + remaining -= grown; + continue; + } + const take = @min(remaining, block_size); + try self.transfers.append(self.allocator, .{ + .destination = destination, + .landing = 0, + .writer_mask = writer_mask, + .destination_offset = destination_offset, + .len = take, + }); + destination_offset += take; + remaining -= take; + } + } + } + + /// Whether a host job's transfers address its read from byte zero + /// to `read_len` in order, each source byte once: then the read + /// can scatter straight into the destinations. Overlapping or + /// duplicate parts address some source bytes twice and go through + /// blocks instead. + fn coversReadInOrder(transfers: []const Plan.Transfer, read_len: usize) bool { + var next: usize = 0; + for (transfers) |transfer| { + if (transfer.landing != next) return false; + next += transfer.len; + } + return next == read_len; + } + }; + fn blockCount(len: usize, block_size: usize) usize { + return len / block_size + @intFromBool(len % block_size != 0); + } + + /// The source bytes every job of `part` reads whole, against one read + /// (`Config.gather_limit`, at most a request). A part whose source runs + /// are back to back reads only its job's range: 0. Any other counts its + /// whole source range, widened to the widest alignment its file may be + /// read at (`Reads.boundOf`): out of order (a broadcast repeats its + /// runs, a transpose permutes them), every job reads that range + /// (`JobCut.appendExtent`); in order with gaps (a slice along an inner + /// axis), a job reads from its first run to its last, which the gaps + /// stretch past any job's length. + fn stridedRead(config: Config, reads: Reads, part: LoadPart) backend.Backend.StridedRead { + const lattice = part.lattice; + const limit = @min(config.request_size, config.gather_limit); + if (lattice.srcTiled()) return .{ .read = 0, .limit = limit }; + const alignment = @max(reads.boundOf(part.source.file_uri), 1); + const start = part.source.offset + lattice.src_offset; + const end = start + lattice.spans().?.src; + return .{ + .read = std.mem.alignForward(u64, end, alignment) - std.mem.alignBackward(u64, start, alignment), + .limit = limit, + }; + } + + /// The longest tensor range one job may cover: the request size, less + /// the two alignment units its widened read can add, so a widened job + /// still fits a request (a credit's slot). `backend.requestSize` refuses + /// a request that cannot hold them. + fn maximumJobLen(request_size: usize, alignment: usize) usize { + std.debug.assert(request_size != 0); + if (alignment == 0) return request_size; + std.debug.assert(std.math.isPowerOfTwo(alignment) and 2 * alignment < request_size); + return request_size - 2 * alignment; + } + + /// Part indices sorted by file URI, delivery class + /// (`Destination.Delivery`), offset, size and index: the planner's + /// input, taken one file group at a time (`fileGroupEnd`), each class + /// in file order. + fn sortedPartOrder(allocator: std.mem.Allocator, parts: []const Part) ![]usize { + const order = try allocator.alloc(usize, parts.len); + for (order, 0..) |*index, i| index.* = i; + const SortContext = struct { + parts: []const Part, + + fn lessThan(ctx: @This(), lhs: usize, rhs: usize) bool { + const left = ctx.parts[lhs]; + const right = ctx.parts[rhs]; + const uri_order = std.mem.order(u8, left.source.file_uri, right.source.file_uri); + if (uri_order != .eq) return uri_order == .lt; + const left_delivery = left.destination.delivery(); + const right_delivery = right.destination.delivery(); + if (left_delivery != right_delivery) return @backingInt(left_delivery) < @backingInt(right_delivery); + const left_offset = left.fileRange().start; + const right_offset = right.fileRange().start; + if (left_offset != right_offset) return left_offset < right_offset; + if (left.lattice.width != right.lattice.width) return left.lattice.width < right.lattice.width; + return lhs < rhs; + } + }; + std.mem.sort(usize, order, SortContext{ .parts = parts }, SortContext.lessThan); + return order; + } + + /// The end of the file group that begins at `order[start]`. + fn fileGroupEnd(parts: []const Part, order: []const usize, start: usize) usize { + const uri = parts[order[start]].source.file_uri; + var end = start + 1; + while (end < order.len and std.mem.eql(u8, uri, parts[order[end]].source.file_uri)) : (end += 1) {} + return end; + } +}; + +/// A strict FIFO of published plans. A submission holds one plan per source +/// file, published as soon as that file is planned, so a plan reaches the +/// queue behind every earlier plan of its own submission and of every +/// earlier one (submissions publish and seal on one task). Within a plan, +/// jobs are handed out in the planned order (file order); a plan leaves the +/// queue with its last claim or through `fail`. +/// A plan is queued only with jobs left and its completion units are added +/// in the same critical section, so a queued plan's batch still holds at +/// least one unit and can never be freed under the scheduler. +const Scheduler = struct { + /// A claimed job: its position in the plan that owns it. The claim holds + /// one of the batch's completion units until the request releases it. + const Claim = struct { + plan: *Plan, + index: usize, + }; + + allocator: std.mem.Allocator, + /// Plans with unclaimed jobs in publish order. + queue: std.Deque(*Plan) = .empty, + /// Set by `fail`: `publish` refuses plans (`Canceled`) and `waitForWork` + /// returns false. + stopping: bool = false, + mutex: std.Io.Mutex = .init, + condition: std.Io.Condition = .init, + + fn init(allocator: std.mem.Allocator) Scheduler { + return .{ .allocator = allocator }; + } + + fn deinit(self: *Scheduler) void { + // Every batch was awaited (and so claimed out or retired) first. + std.debug.assert(self.queue.len == 0); + self.queue.deinit(self.allocator); + self.* = undefined; + } + + /// Publishes one plan of an open submission behind every earlier plan. + /// A plan without jobs never joins the queue. `Canceled` once the + /// scheduler failed (`fail`). + fn publish(self: *Scheduler, io: std.Io, batch: *Batch, plan: *Plan) !void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + if (self.stopping) return error.Canceled; + const job_count = plan.jobs.len; + try batch.plans.ensureUnusedCapacity(batch.arena.allocator(), 1); + if (job_count != 0) try self.queue.ensureUnusedCapacity(self.allocator, 1); + // Nothing below fails: the plan, its units and the queue entry + // appear together. + plan.batch = batch; + batch.plans.appendAssumeCapacity(plan); + _ = batch.remaining.fetchAdd(job_count, .acq_rel); + if (job_count != 0) { + self.queue.pushBackAssumeCapacity(plan); + self.condition.broadcast(io); + } + } + + /// Stops claims and publishes, wakes every worker waiting for work, and + /// retires the unclaimed units of every queued plan so its batch still + /// reaches `done` through its claimed requests (an open one through its + /// seal and sentinel as well). Claims and this pass both move cursors + /// under the mutex, so they partition the jobs exactly: every claimed + /// job keeps its unit with its worker. Each queued plan still holds an + /// unclaimed unit of its batch, so only the retire of a batch's last + /// queued plan can complete it, and nothing touches the batch after that. + fn fail(self: *Scheduler, io: std.Io) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.stopping = true; + self.condition.broadcast(io); + while (self.queue.popFront()) |plan| { + const unclaimed = plan.jobs.len - plan.cursor; + plan.cursor = plan.jobs.len; + plan.batch.releaseUnits(unclaimed); + } + } + + /// Hands out the head plan's next job; the plan leaves the queue with + /// its last one. Null only when nothing is queued: a queued plan always + /// has a job left. + fn claim(self: *Scheduler, io: std.Io) ?Claim { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + const plan = self.queue.front() orelse return null; + const index = plan.cursor; + std.debug.assert(index < plan.jobs.len); + plan.cursor += 1; + if (plan.cursor == plan.jobs.len) _ = self.queue.popFront(); + return .{ .plan = plan, .index = index }; + } + + fn waitForWork(self: *Scheduler, io: std.Io) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (!self.stopping and self.queue.len == 0) { + self.condition.waitUncancelable(io, &self.mutex); + } + return !self.stopping; + } +}; + +/// A claimed source job and its completion state. The plan owns its storage. +const ReadRequest = struct { + const Scratch = struct { + allocator: std.mem.Allocator, + /// A read's ranges: one per block of a job (`readBlocks`), or a + /// scattered read's, at most one vectored read's worth (`Scatter`). + iovecs: [][]u8, + /// A landing job's reads that do not scatter (`landJob`), allocated + /// on first use: as large as a job's blocks, page aligned for + /// O_DIRECT. + bounce: []align(std.heap.page_size_min) u8 = &.{}, + bounce_capacity: usize, + + fn init(allocator: std.mem.Allocator, maximum_blocks: usize, block_size: usize) !Scratch { + return .{ + .allocator = allocator, + .iovecs = try allocator.alloc([]u8, @max(maximum_blocks, max_positional_iovecs)), + .bounce_capacity = maximum_blocks * block_size, + }; + } + + /// The ranges a `Scatter` gathers before it reads. + fn scatterIovecs(self: *const Scratch) [][]u8 { + return self.iovecs[0..max_positional_iovecs]; + } + + fn bounceBuffer(self: *Scratch, len: usize) ![]u8 { + std.debug.assert(len <= self.bounce_capacity); + if (self.bounce.len == 0) self.bounce = try self.allocator.alignedAlloc(u8, .fromByteUnits(std.heap.page_size_min), self.bounce_capacity); + return self.bounce[0..len]; + } + + fn deinit(self: *Scratch) void { + self.allocator.free(self.bounce); + self.allocator.free(self.iovecs); + self.* = undefined; + } + }; + + loader: *Loader, + plan: *Plan, + /// The job's index in its plan, which holds this request at the same + /// index. + index: usize, + /// The request's lifecycle credit: its slot of blocks (`Credits`), which + /// only this request reads into and transfers from until it ends. + slot: u32, + /// The scheduling sentinel and one reference per DMA piece queued; the + /// last release gives the credit back and finishes the job. + pending: std.atomic.Value(usize) = .init(1), + /// Awake-clock nanoseconds of the enqueue; 0 until then. + enqueued_ns: u64 = 0, + + /// The slot of a job that was never claimed: nothing pending, so the + /// retirement checks hold. + const idle: ReadRequest = .{ + .loader = undefined, + .plan = undefined, + .index = undefined, + .slot = undefined, + .pending = .init(0), + }; + + /// Takes the claimed job's request context with the worker's credit. + /// The claim holds the batch's completion unit; the request's final + /// reference drop releases it. + fn init(loader: *Loader, claim: Scheduler.Claim, slot: u32) *ReadRequest { + const self = &claim.plan.requests[claim.index]; + self.* = .{ + .loader = loader, + .plan = claim.plan, + .index = claim.index, + .slot = slot, + }; + return self; + } + + fn run(self: *ReadRequest, scratch: *Scratch) !void { + const loader = self.loader; + const io = loader.io; + const block_size = loader.plan_config.block_size; + const job = self.plan.jobs[self.index]; + const transfers = self.plan.transfersOf(job); + // The first worker to touch a destination creates its state; a + // host-resident one's is normally created ahead by the allocation + // task. + var previous: ?*Destination = null; + for (transfers) |transfer| { + const destination = transfer.destination; + if (destination == previous) continue; + previous = destination; + const init_started = awakeNs(io); + _ = destination.ensureState(loader) catch |err| { + loader.recordError(err, "initialize tensor {s} from {s}, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); + return err; + }; + _ = loader.metrics.tensor_init_ns.fetchAdd(awakeNs(io) -| init_started, .monotonic); + } + const blocks = loader.credits.slotBlocks(self.slot)[0..job.blockLen(block_size)]; + switch (job.kind) { + .dma_read => |read| try self.readBlocks(scratch, blocks, read), + .dma_land => |land| try landJob(loader, self.plan, land, blocks, scratch), + // Reads nothing: its transfers read the zero block. + .dma_fill => return Dma.enqueue(self, loader.blocks[0..1]), + // A host job's worker writes its destinations itself, so the job + // ends with the worker's scheduling reference; its slot's blocks, + // if it reads through them, are done when this returns. + .host_direct => |read| { + var scatter: Scatter = .{ + .loader = loader, + .slot = self.plan.source_slot, + .iovecs = scratch.scatterIovecs(), + .file_offset = job.file_offset, + }; + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + // The planner checked that the transfers cover the read + // in order, each source byte once. + try scatter.add(tensor.hostLanding(transfer)); + } + try scatter.flush(); + std.debug.assert(scatter.file_offset == job.file_offset + read.len); + }, + .host_bounce => |read| { + try self.readBlocks(scratch, blocks, read); + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + // The last block holds stale bytes past the read, which + // no bounds check would stop a copy from taking. + std.debug.assert(transfer.landing + transfer.len <= read.len); + const piece = blockPiece(blocks, block_size, transfer.landing, transfer.len); + std.debug.assert(piece.len == transfer.len); + @memcpy(tensor.hostLanding(transfer), piece); + } + }, + .host_fill => { + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + @memset(tensor.hostLanding(transfer), 0); + } + return; + }, + } + for (transfers) |transfer| transfer.destination.recordRead(transfer.len); + if (job.kind == .dma_read or job.kind == .dma_land) try Dma.enqueue(self, blocks); + } + + /// Reads the job's `read` into `blocks`, its slot's, in file order. + fn readBlocks(self: *ReadRequest, scratch: *Scratch, blocks: []const host_memory.Block, read: Plan.Job.Read) !void { + const loader = self.loader; + const block_size = loader.plan_config.block_size; + const iovecs = scratch.iovecs[0..blocks.len]; + for (iovecs, blocks, 0..) |*iovec, block, block_index| { + iovec.* = block[0..@min(block_size, read.len - block_index * block_size)]; + } + try readSlot(loader, self.plan.source_slot, iovecs, self.plan.jobs[self.index].file_offset, read.minimum_len); + } + + /// Releases a DMA piece or the scheduling reference. The final release + /// gives the credit back and may let the awaiting task free this request + /// and everything its batch owns. + fn release(self: *ReadRequest) void { + const previous = self.pending.fetchSub(1, .acq_rel); + std.debug.assert(previous > 0); + if (previous != 1) return; + + // Locals first: releasing the batch unit may complete the batch + // that owns this request, so nothing is touched after it. + const loader = self.loader; + const batch = self.plan.batch; + if (self.enqueued_ns != 0) { + _ = loader.metrics.dma_stage_ns.fetchAdd(awakeNs(loader.io) -| self.enqueued_ns, .monotonic); + } + loader.credits.release(loader.io, self.slot); + batch.releaseUnits(1); + } +}; + +/// Reads `[file_offset, file_offset + minimum)`, and possibly more, into +/// `buffers` in order, consuming them as it reads; bytes past `minimum` may +/// stay unread. The io takes as many buffers per call as it can (VFS's +/// direct read `IOV_MAX`, std's and zio's 8, a remote read all); a short +/// read resumes where it stopped. Reading stops once `minimum` bytes are +/// in, so the file may end only past `minimum`: the bytes after it are +/// padding a caller added to align the read, and the read that would fetch +/// them can start unaligned, which a file read directly refuses. Returns +/// the bytes read. +fn readFilePositionalAllV( + io: std.Io, + file: std.Io.File, + buffers: [][]u8, + file_offset: u64, + minimum: u64, +) !u64 { + var index: usize = 0; + var completed: u64 = 0; + while (completed < minimum) { + std.debug.assert(index < buffers.len); + const bytes_read = try file.readPositional(io, buffers[index..], file_offset + completed); + if (bytes_read == 0) return error.UnexpectedEndOfFile; + completed += bytes_read; + var rest = bytes_read; + while (index < buffers.len and rest >= buffers[index].len) : (index += 1) rest -= buffers[index].len; + if (rest != 0) buffers[index] = buffers[index][rest..]; + } + return completed; +} + +/// Every worker read goes through here, under a read permit: a failure is +/// recorded with the file, since a landing job's extents can come from files +/// other than its plan's. Once the loader failed, returns its first error. +/// Consumes `iovecs` (`readFilePositionalAllV`). +fn readSlot(loader: *Loader, slot: *SourceSlot, iovecs: [][]u8, file_offset: u64, minimum_len: usize) !void { + const io = loader.io; + const file = slot.ensure(io) catch |err| { + loader.recordError(err, "open {s}", .{slot.uri}); + return err; + }; + loader.read_permits.waitUncancelable(io); + defer loader.read_permits.post(io); + if (loader.first_error.get()) |err| return err; + const read_started = awakeNs(io); + const read_result = readFilePositionalAllV(io, file, iovecs, file_offset, minimum_len); + _ = loader.metrics.read_ns.fetchAdd(awakeNs(io) -| read_started, .monotonic); + const bytes_read = read_result catch |err| { + loader.recordError(err, "read {s} at {d}, {d} bytes at least", .{ slot.uri, file_offset, minimum_len }); + return err; + }; + _ = loader.metrics.read_operations.fetchAdd(1, .monotonic); + _ = loader.metrics.read_bytes.fetchAdd(@intCast(bytes_read), .monotonic); +} + +/// Reads a landing job's extents and lands their items' runs in its blocks +/// (`Plan.Item`, `walkItem`). An extent marked for it scatters +/// straight into the blocks, `max_positional_iovecs` ranges per call; any +/// other lands in the worker's bounce buffer and is copied from there: a +/// direct read, whose ranges must be aligned, and a gather read whose +/// source runs are out of order or repeat (a transpose, a broadcast). +fn landJob( + loader: *Loader, + plan: *const Plan, + land: Plan.Job.Land, + blocks: []const host_memory.Block, + scratch: *ReadRequest.Scratch, +) !void { + if (land.zeroed) { + for (blocks) |block| @memset(block[0..loader.plan_config.block_size], 0); + } + for (plan.extents[land.extent_start..][0..land.extent_len]) |extent| { + try landExtent(loader, extent, plan.items[extent.item_start..][0..extent.item_len], plan.landings, blocks, scratch); + } +} + +fn landExtent( + loader: *Loader, + extent: Plan.Extent, + items: []const Plan.Item, + landings: []const u64, + blocks: []const host_memory.Block, + scratch: *ReadRequest.Scratch, +) !void { + const block_size = loader.plan_config.block_size; + if (extent.scatter) { + std.debug.assert(extent.len == extent.minimum_len); + var scatter: Scatter = .{ + .loader = loader, + .slot = extent.source_slot, + .iovecs = scratch.scatterIovecs(), + .file_offset = extent.file_offset, + .blocks = blocks, + }; + for (items) |item| try walkItem(item, landings, &scatter); + try scatter.flush(); + std.debug.assert(scatter.file_offset == extent.file_offset + extent.len); + return; + } + const bounce = try scratch.bounceBuffer(extent.len); + var buffers = [_][]u8{bounce}; + try readSlot(loader, extent.source_slot, &buffers, extent.file_offset, extent.minimum_len); + var sink: CopySink = .{ .bounce = bounce, .file_offset = extent.file_offset, .blocks = blocks, .block_size = block_size }; + for (items) |item| try copyItem(item, landings, &sink); +} + +/// Hands `sink` every run of `item` as `place(file offset, landing, len)`, +/// in run order, cut where the destination's box changes +/// (`ShardGrid.runEnd`). +fn walkItem(item: Plan.Item, landings: []const u64, sink: anytype) !void { + const table = landings[item.landing_start..][0..item.destination.grid.box_count]; + try walkRuns(item.lattice, item.first, item.end, item.clip_start, item.clip_end, &item.destination.grid, table, sink); +} + +/// `walkItem` for runs `first` to `end` of `lattice`, clipped to the +/// destination range `[clip_start, clip_end)`. +fn walkRuns(lattice: backend.Lattice, first: u64, end: u64, clip_start: u64, clip_end: u64, grid: *const ShardGrid, table: []const u64, sink: anytype) !void { + if (first >= end) return; + var runs: backend.Lattice.Cursor = .init(lattice, first, end); + while (runs.next()) |run| { + var at = @max(run.dst, clip_start); + const stop = @min(run.dst + lattice.width, clip_end); + var source = run.src + (at - run.dst); + while (at < stop) { + const piece_end = grid.runEnd(at, stop); + const box, const box_offset = grid.locate(at); + try sink.place(source, table[box] +% box_offset, piece_end - at); + source += piece_end - at; + at = piece_end; + } + } +} + +/// `walkItem` from the bounce buffer, a stretch at a time where that pays: +/// a stretch of the destination landing in one piece, inside one run of +/// the grid and one block, has its whole runs copied by +/// `Lattice.copyRuns`, and only the runs its ends cut go one at a time. A +/// stretch costs a few bisections, so items with few runs per stretch (a +/// part of a few wide runs, a grid that cuts every row) are walked. +fn copyItem(item: Plan.Item, landings: []const u64, sink: *CopySink) !void { + const lattice = item.lattice; + const grid = &item.destination.grid; + if (lattice.dims.len == 0 or item.first >= item.end) return walkItem(item, landings, sink); + const start = @max(item.clip_start, lattice.at(item.first).dst); + const stop = @min(item.clip_end, lattice.at(item.end - 1).dst + lattice.width); + if (start >= stop) return; + const stretches = grid.runCount(start, stop) + (stop - start) / sink.block_size + 1; + if (item.end - item.first < 16 * stretches) return walkItem(item, landings, sink); + const table = landings[item.landing_start..][0..grid.box_count]; + const source = @intFromPtr(sink.bounce.ptr) -% @as(usize, @intCast(sink.file_offset)); + var at = start; + while (at < stop) { + const box, const box_offset = grid.locate(at); + const landing: usize = @intCast(table[box] +% box_offset); + const in_block = landing % sink.block_size; + const stretch_end = @min(grid.runEnd(at, stop), at + (sink.block_size - in_block)); + // The address logical offset zero would have, were the whole + // destination laid out like this stretch. + const destination = @intFromPtr(sink.blocks[landing / sink.block_size].ptr) + in_block -% @as(usize, @intCast(at)); + const touched = @max(item.first, lattice.firstEndingAfter(at)); + const whole = @max(item.first, lattice.firstStartingFrom(at)); + const whole_end = @min(item.end, lattice.firstEndingAfter(stretch_end)); + const touched_end = @min(item.end, lattice.firstStartingFrom(stretch_end)); + if (whole < whole_end) { + try walkRuns(lattice, touched, whole, at, stretch_end, grid, table, sink); + lattice.copyRuns(whole, whole_end, destination, source); + try walkRuns(lattice, whole_end, touched_end, at, stretch_end, grid, table, sink); + } else { + try walkRuns(lattice, touched, touched_end, at, stretch_end, grid, table, sink); + } + at = stretch_end; + } +} + +/// The ranges a `Scatter` gathers before it reads, as many as one direct +/// read takes. +const max_positional_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 else std.posix.IOV_MAX; + +/// A read scattered over memory ranges as they come, in file order and +/// back to back from `file_offset` of `slot`'s file: a range that continues +/// the previous one in memory merges with it, and one call reads at most +/// `iovecs.len` ranges. Fed destination ranges directly (`add`), or the runs +/// of a walk landing in `blocks` (`place`). Every byte read must exist: a +/// scattered read has no alignment padding. +const Scatter = struct { + loader: *Loader, + slot: *SourceSlot, + iovecs: [][]u8, + count: usize = 0, + /// Bytes the pending ranges cover. + pending: usize = 0, + /// The file offset of the first byte not read yet. + file_offset: u64, + blocks: []const host_memory.Block = &.{}, + + fn add(self: *Scatter, range: []u8) !void { + if (self.count != 0) { + const last = &self.iovecs[self.count - 1]; + if (last.ptr + last.len == range.ptr) { + last.len += range.len; + self.pending += range.len; + return; + } + } + if (self.count == self.iovecs.len) try self.flush(); + self.iovecs[self.count] = range; + self.count += 1; + self.pending += range.len; + } + + /// A run's landing in the blocks, split at block boundaries. + fn place(self: *Scatter, source: u64, landing: u64, len: u64) !void { + std.debug.assert(source == self.file_offset + self.pending); + var at: usize = @intCast(landing); + var rest: usize = @intCast(len); + while (rest != 0) { + const piece = blockPiece(self.blocks, self.loader.plan_config.block_size, at, rest); + try self.add(piece); + at += piece.len; + rest -= piece.len; + } + } + + fn flush(self: *Scatter) !void { + if (self.count == 0) return; + try readSlot(self.loader, self.slot, self.iovecs[0..self.count], self.file_offset, self.pending); + self.file_offset += self.pending; + self.count = 0; + self.pending = 0; + } +}; + +/// Runs copied from a read in the bounce buffer, which starts at +/// `file_offset`. +const CopySink = struct { + bounce: []const u8, + file_offset: u64, + blocks: []const host_memory.Block, + block_size: usize, + + fn place(self: *CopySink, source: u64, landing: u64, len: u64) !void { + var at: usize = @intCast(landing); + var rest = self.bounce[@intCast(source - self.file_offset)..][0..@intCast(len)]; + while (rest.len != 0) { + const piece = blockPiece(self.blocks, self.block_size, at, rest.len); + @memcpy(piece, rest[0..piece.len]); + rest = rest[piece.len..]; + at += piece.len; + } + } +}; + +/// The bytes from offset `at` of `blocks` taken end to end, `block_size` +/// bytes each, up to `len` of them or the end of that block. +fn blockPiece(blocks: []const host_memory.Block, block_size: usize, at: usize, len: usize) []u8 { + const in_block = at % block_size; + return blocks[at / block_size][in_block..][0..@min(len, block_size - in_block)]; +} + +/// The DMA stage, over the loader's pumps (`Loader.pumps`): workers queue a +/// request's transfers per device, a pump submits them under the device's +/// in-flight budget, and their callbacks release the request. +const Dma = struct { + /// One DMA piece: `data`, in its request's slot, for `target` at + /// `destination_offset`. The piece holds one of its request's + /// references until its callback. Replicas share their source bytes: + /// copying them to another CPU socket first was rejected on MI300X, + /// where raw local and remote H2D both reached ~49-50 GiB/s per GPU + /// while one CPU thread copied across sockets at 5-10 GiB/s. + const Piece = struct { + target: *DestinationState.Target, + request: *ReadRequest, + data: []const u8, + destination_offset: usize, + }; + + /// Pieces ready for one device, submitted in arrival order so the + /// oldest requests and submissions complete first. Owned by the + /// device's `DevicePump.mutex`. + const ReadyQueue = std.Deque(Piece); + + /// One device's submission state. Its pump runs on whichever thread + /// finds work -- a worker after enqueueing, a ready callback after a + /// completion -- one at a time per device (`pumping`), and shares + /// nothing with the other devices' pumps. Per-device pumps load + /// replicated DeepSeek on four GB300 in 4.68 s, against 5.41 s with one + /// pump for every device, at unchanged pinned high-water. A dedicated + /// task per device took 5.0-5.4 s with a wake per completion (a futex + /// round trip per piece), or 4.4-5.1 s with half-budget wake hysteresis. + const DevicePump = struct { + mutex: std.Io.Mutex = .init, + queue: ReadyQueue = .empty, + /// In-flight DMA bytes and submissions; owned by `mutex`. + active_bytes: usize = 0, + active_pieces: usize = 0, + /// True while one thread runs this device's pump; owned by `mutex`. + /// One submitter per device is a correctness requirement: + /// `Target.nextIsLast` flags a tensor's last piece in submission + /// order. + pumping: bool = false, + /// Contexts whose callback fired, for the next pump or a batch's + /// retirement to destroy, never the event's own callback: an + /// intrusive stack through `EventContext.next_retired`, owned by + /// `mutex`. Destroying an event from the next pump instead of at its + /// batch's retirement bounds live PJRT events by the DMA width plus + /// one pump batch rather than by a submission's transfer count; + /// every shipped plugin accepted destruction outside the event's own + /// callback (16,384 fired events on two B70, CUDA and ROCm alike). + retired: ?*EventContext = null, + + /// Whether the next queued piece may go. The budget is in bytes, + /// not submissions: `dma_calibration.depth` blocks, and a piece is + /// one tensor's slice of a block, so a model of small tensors needs + /// many more pieces in flight to keep the same bytes moving: eight + /// average DeepSeek pieces occupy only ~18 MiB against the 128 MiB + /// budget of 16 MiB blocks. The piece that crosses the budget is + /// admitted: a device with room always has a piece in flight. + fn hasRoom(self: *const DevicePump, budget_bytes: usize) bool { + return self.active_bytes < budget_bytes and self.active_pieces < max_dma_pieces_per_device; + } + + /// `mutex`. Destroys every retired event. + fn destroyRetired(self: *DevicePump) void { + while (self.retired) |ctx| { + self.retired = ctx.next_retired; + ctx.next_retired = null; + ctx.destroyEvent(); + } + } + }; + + const EventContext = struct { + request: *ReadRequest, + /// Null once destroyed: by a pump or a batch's retirement after its + /// callback ran, or at once by a failed `onReady`. + pjrt_event: ?*pjrt.Event, + device_index: usize, + /// Bytes this submission holds against the device's in-flight budget. + len: usize, + /// Awake-clock nanoseconds of the submission. + submitted_ns: u64 = 0, + /// Link of the device pump's `retired`; owned by that pump's mutex. + next_retired: ?*EventContext = null, + + /// Pump mutex, unless no callback was registered. Destroys the + /// event, once. The platform is reached through the request, and + /// only when there is something to destroy: a context whose event + /// is null never touches it. + fn destroyEvent(self: *EventContext) void { + if (self.pjrt_event) |event| event.deinit(self.request.loader.platform.pjrt_api); + self.pjrt_event = null; + } + }; + + fn lockAllPumps(loader: *Loader) void { + for (loader.pumps) |*device_pump| device_pump.mutex.lockUncancelable(loader.io); + } + + fn unlockAllPumps(loader: *Loader) void { + var index = loader.pumps.len; + while (index > 0) { + index -= 1; + loader.pumps[index].mutex.unlock(loader.io); + } + } + + /// Retires the contexts of a done batch from the awaiting task, after + /// their callbacks fired, exactly as `pjrt.Event.await` does: drains + /// every pump's `retired`, whichever batch its contexts belong to, and + /// checks that every request completed. Runs with every pump locked so + /// an `abortReady` still iterating queued entries or a pump draining its + /// `retired` cannot race the free that follows. + fn retireBatch(loader: *Loader, batch: *Batch) void { + std.debug.assert(batch.done.isSet()); + lockAllPumps(loader); + defer unlockAllPumps(loader); + for (loader.pumps) |*device_pump| device_pump.destroyRetired(); + for (batch.plans.items) |plan| { + for (plan.requests) |*request| std.debug.assert(request.pending.load(.acquire) == 0); + } + } + + /// Hands a context whose callback fired to its device's next pump. Under + /// that pump's mutex, so the batch's retirement sees it before the batch + /// is freed. + fn retireEvent(loader: *Loader, ctx: *EventContext) void { + const device_pump = &loader.pumps[ctx.device_index]; + device_pump.mutex.lockUncancelable(loader.io); + ctx.next_retired = device_pump.retired; + device_pump.retired = ctx; + device_pump.mutex.unlock(loader.io); + } + + /// Queues every piece of the request's job before pumping: queuing piece + /// by piece costs roughly 69k-79k mutex and pump trips per DeepSeek load, + /// which coalesced reads alone do not remove. A transfer reads `len` + /// bytes from its `landing` of `blocks` taken end to end, inside one + /// block, once for each of its writers: the devices the planner planned + /// for (both come from the sharding's canonical device order). + fn enqueue(request: *ReadRequest, blocks: []const host_memory.Block) !void { + const loader = request.loader; + const block_size = loader.plan_config.block_size; + const transfers = request.plan.transfersOf(request.plan.jobs[request.index]); + var counts: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + var pieces: usize = 0; + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + pieces += @popCount(transfer.writer_mask); + var mask = transfer.writer_mask; + while (mask != 0) { + const writer_index: usize = @intCast(@ctz(mask)); + mask &= mask - 1; + counts[tensor.targets[writer_index].device_index] += 1; + } + } + const enqueued_ns = awakeNs(loader.io); + // Every pump is held while the request's pieces land, and every + // queue is reserved before any changes, so the request is either + // fully queued or not queued at all. + lockAllPumps(loader); + { + errdefer unlockAllPumps(loader); + // A failure recorded before these locks may have drained the + // queues already (`abortReady`), and nothing would complete what + // this queued; one recorded after them drains it. + if (loader.first_error.get()) |err| return err; + for (loader.pumps, counts[0..loader.pumps.len]) |*device_pump, count| { + try device_pump.queue.ensureUnusedCapacity(loader.allocator, count); + } + } + // Every piece holds the request, and with it the slot, until its + // callback. + _ = request.pending.fetchAdd(pieces, .acq_rel); + request.enqueued_ns = enqueued_ns; + for (transfers) |transfer| { + // The planner cut the transfers to the job's blocks. + const data = blockPiece(blocks, block_size, transfer.landing, transfer.len); + std.debug.assert(data.len == transfer.len); + const tensor = transfer.destination.state.readyValue().?; + var mask = transfer.writer_mask; + while (mask != 0) { + const writer_index: usize = @intCast(@ctz(mask)); + mask &= mask - 1; + const target = &tensor.targets[writer_index]; + loader.pumps[target.device_index].queue.pushBackAssumeCapacity(.{ + .target = target, + .request = request, + .data = data, + .destination_offset = transfer.destination_offset, + }); + } + } + unlockAllPumps(loader); + for (counts[0..loader.pumps.len], 0..) |count, device_index| { + if (count != 0) requestPump(loader, device_index); + } + } + + /// Runs the device's pump on this thread unless one is already running. + fn requestPump(loader: *Loader, device_index: usize) void { + const device_pump = &loader.pumps[device_index]; + device_pump.mutex.lockUncancelable(loader.io); + if (device_pump.pumping or loader.first_error.get() != null) { + device_pump.mutex.unlock(loader.io); + return; + } + device_pump.pumping = true; + device_pump.mutex.unlock(loader.io); + pump(loader, device_index); + } + + /// Submits from the device's queue, in arrival order, until the device + /// has no room or nothing is queued. Any queued piece can go: nothing + /// waits for another piece. Exactly one pump runs per device at a time, + /// see `DevicePump.pumping`. + fn pump(loader: *Loader, device_index: usize) void { + const device_pump = &loader.pumps[device_index]; + while (true) { + var selected: ?Piece = null; + device_pump.mutex.lockUncancelable(loader.io); + device_pump.destroyRetired(); + if (loader.first_error.get() == null) { + if (!device_pump.hasRoom(loader.dma_budget_bytes)) { + _ = loader.metrics.pump_stops_full.fetchAdd(1, .monotonic); + } else if (device_pump.queue.popFront()) |piece| { + device_pump.active_bytes += piece.data.len; + device_pump.active_pieces += 1; + selected = piece; + } else { + _ = loader.metrics.pump_stops_empty.fetchAdd(1, .monotonic); + } + } + if (selected == null) device_pump.pumping = false; + device_pump.mutex.unlock(loader.io); + if (selected) |piece| submitOne(loader, piece) else return; + } + } + + fn submitOne(loader: *Loader, piece: Piece) void { + const device_index = piece.target.device_index; + const len = piece.data.len; + submitTransfer(loader, piece) catch |err| { + loader.recordError(err, "submit DMA transfer: device={d}, destination_offset={d}, bytes={d}", .{ device_index, piece.destination_offset, len }); + finishPiece(loader, device_index, len, null, piece.request); + }; + } + + fn submitTransfer(loader: *Loader, piece: Piece) !void { + const api = loader.platform.pjrt_api; + const target = piece.target; + const is_last = target.nextIsLast(piece.data.len); + const submit_started = awakeNs(loader.io); + const event = try target.manager.transferData( + api, + 0, + piece.data, + @intCast(piece.destination_offset), + is_last, + ); + target.noteSubmitted(piece.data.len); + + // The plan holds one event slot per planned submission; the batch + // owns it. + const plan = piece.request.plan; + const event_index = plan.events_used.fetchAdd(1, .monotonic); + std.debug.assert(event_index < plan.events.len); + const ctx = &plan.events[event_index]; + ctx.* = .{ + .request = piece.request, + .pjrt_event = event, + .device_index = target.device_index, + .len = piece.data.len, + .submitted_ns = submit_started, + }; + + _ = loader.metrics.dma_submissions.fetchAdd(1, .monotonic); + event.onReady(api, EventContext, struct { + fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { + const request = ctx_.request; + const loader_ = request.loader; + const api_ = loader_.platform.pjrt_api; + const device_index = ctx_.device_index; + const len = ctx_.len; + _ = loader_.metrics.dma_piece_ns.fetchAdd(awakeNs(loader_.io) -| ctx_.submitted_ns, .monotonic); + // The shipped plugins resolve this event without an error + // whatever happened to the copy (the C API wrapper sets the + // promise with an OK status); kept for a plugin that reports. + // The callback owns the error. + if (err) |pjrt_error| { + loader_.recordError(pjrt_error.getCode(api_).toApiError(), "complete DMA transfer: device={d}, bytes={d}, message={s}", .{ device_index, len, pjrt_error.getMessage(api_) }); + pjrt_error.deinit(api_); + } + finishPiece(loader_, device_index, len, ctx_, request); + } + }.call, ctx) catch |err| { + event.awaitRaw(api) catch {}; + ctx.destroyEvent(); + return err; + }; + _ = loader.metrics.dma_submit_ns.fetchAdd(awakeNs(loader.io) -| submit_started, .monotonic); + } + + /// Ends a submitted piece: frees its room, which may pump on this thread; + /// then retires a fired context, after that pump, so its event is never + /// destroyed inside its own callback; then releases the request last, + /// since that release may complete and free the batch. + fn finishPiece(loader: *Loader, device_index: usize, len: usize, fired: ?*EventContext, request: *ReadRequest) void { + eventCompleted(loader, device_index, len); + if (fired) |ctx| retireEvent(loader, ctx); + request.release(); + } + + fn eventCompleted(loader: *Loader, device_index: usize, len: usize) void { + const device_pump = &loader.pumps[device_index]; + device_pump.mutex.lockUncancelable(loader.io); + std.debug.assert(device_pump.active_bytes >= len); + std.debug.assert(device_pump.active_pieces > 0); + device_pump.active_bytes -= len; + device_pump.active_pieces -= 1; + device_pump.mutex.unlock(loader.io); + // A ready callback can be the first place an asynchronous PJRT error + // becomes visible. Once outside the pump lock, retire every queued + // piece so request lifecycles cannot wait forever on entries that + // the failed pumps will no longer submit. + if (loader.first_error.get() != null) + abortReady(loader) + else + requestPump(loader, device_index); + } + + /// Force-completes every queued piece, one device at a time. A + /// completion here may finish its batch, but a batch still has an entry + /// queued until its last one is popped, and it is retired with every + /// pump locked, so no queued entry is touched after its request ends. + fn abortReady(loader: *Loader) void { + for (loader.pumps) |*device_pump| { + device_pump.mutex.lockUncancelable(loader.io); + while (device_pump.queue.popFront()) |piece| { + piece.request.release(); + } + device_pump.mutex.unlock(loader.io); + } + } +}; + +const DestinationState = struct { + /// One destination buffer of a tensor. PJRT makes the buffer ready once + /// every transfer submitted to it has completed and one of them carried + /// the last-transfer flag, whatever their completion order: the flag + /// only closes the buffer to further calls. The pump, the only + /// submitter, therefore flags the submission that completes the + /// placement's bytes, and no piece ever waits for another one. + /// Flagging the highest-offset piece instead makes a tail wait for its + /// prefixes, which can deadlock a sharded multi-device load: tails can + /// hold every lifecycle credit while their prefixes stay unclaimed, + /// whatever the pinned memory. This follows XLA's + /// CommonAsyncHostToDeviceTransferManager: readiness requires zero + /// in-flight transfers and a last-call flag, not destination-offset order. + const Target = struct { + manager: *pjrt.AsyncHostToDeviceTransferManager, + device_index: usize, + /// Bytes of the placement on this device. The pieces partition them: + /// those of every part, from whichever file and plan, and of the + /// zero fill. + total: usize, + /// Owned by the pump; read by the awaiting task once the batch is + /// done and nothing submits any more. + submitted_bytes: usize = 0, + + /// Whether a submission of `len` bytes closes the buffer. + fn nextIsLast(self: *const Target, len: usize) bool { + std.debug.assert(len != 0 and self.submitted_bytes + len <= self.total); + return self.submitted_bytes + len == self.total; + } + + fn noteSubmitted(self: *Target, len: usize) void { + std.debug.assert(self.submitted_bytes + len <= self.total); + self.submitted_bytes += len; + } + + /// Every byte went out, so the last-transfer flag did too. + fn fullySubmitted(self: *const Target) bool { + return self.submitted_bytes == self.total; + } + }; + + /// A DMA destination's transfer targets, one per device in canonical + /// order; empty for a host-resident one. + targets: []Target = &.{}, + /// A host-resident destination's shards in host memory, which workers + /// write in place, one per device in canonical order; empty for a DMA + /// one. The devices of one box share theirs (`initHost`). + host: [][*]u8 = &.{}, + + /// Creates the destination's buffers and writes the output shell: for a + /// DMA destination, the device buffers and transfer managers, one per + /// device of its grid in canonical order; for a host-resident one, the + /// buffers workers write (`initHost`). Runs once per destination + /// (`LazyOnce`), however many parts feed it, on the allocation task or + /// the worker that first reads or fills for it, which also counts the + /// allocation once. A host destination (`Memory.isHost`) is created in + /// each device's pinned host memory space and takes no device memory, + /// so it is not counted in `allocated_bytes`. + fn init(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { + if (destination.memory.isHost()) return initHost(direct, destination); + const allocator = direct.allocator; + const platform = direct.platform; + const grid = &destination.grid; + const targets = try allocator.alloc(Target, grid.devices.len); + errdefer allocator.free(targets); + + var pjrt_buffers: Buffer.Shards = .empty; + var initialized: usize = 0; + errdefer { + for (targets[0..initialized]) |target| target.manager.deinit(platform.pjrt_api); + for (pjrt_buffers.constSlice()) |buffer| buffer.deinit(platform.pjrt_api); + } + + const shape_spec: pjrt.ShapeSpec = .init( + grid.local.dims(), + pjrtx.bufferTypeFromDtype(grid.local.dtype()), + ); + for (grid.devices, 0..) |device, i| { + const memory = platform.devices[device.id].memory(.default).?; + const manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ + .shape_specs = &.{shape_spec}, + .memory = memory.pjrt_memory, + }); + errdefer manager.deinit(platform.pjrt_api); + const pjrt_buffer = try manager.retrieveBuffer(platform.pjrt_api, 0); + targets[i] = .{ + .manager = manager, + .device_index = device.id, + .total = grid.local.byteSize(), + }; + initialized += 1; + pjrt_buffers.appendAssumeCapacity(pjrt_buffer); + } + for (targets) |target| { + _ = direct.allocated_bytes[target.device_index].fetchAdd(target.total, .monotonic); + } + + destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); + return .{ .targets = targets }; + } + + /// Creates a host-resident destination's buffers, one per device of its + /// grid in canonical order, each in that device's pinned host memory + /// space, and reads their host pointers for the workers. The devices + /// holding one box (`ShardGrid.writers`) share one allocation: every + /// device's shard is a view of it, and the allocation lives until the + /// last view is gone (`SharedHostAllocation`). A transfer is then + /// written once, through its first writer's shard (`hostLanding`). A + /// `host_mapped` shard is always such a view, on the device's default + /// memory space, so kernels read the allocation in place. Every direct + /// target lets a host allocation of one device back a view on another, + /// for transfers and kernels alike: CPU, CUDA and ROCm allocate portable + /// pinned memory (`Buffer.HostAccessible.init`), and on oneAPI four B70 + /// gather from an allocation made in the first one's host memory space, + /// and read it back. A target that could not would need a copy per + /// device. + /// The buffers are complete before any worker writes them, so they + /// report ready from the start; the front end hands them out only once + /// the batch is done. + fn initHost(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { + const allocator = direct.allocator; + const platform = direct.platform; + const api = platform.pjrt_api; + const grid = &destination.grid; + const shard_count = grid.devices.len; + std.debug.assert(shard_count <= Platform.MAX_NUM_DEVICES); + const shards = try allocator.alloc([*]u8, shard_count); + errdefer allocator.free(shards); + const dims = grid.local.dims(); + const element_type = pjrtx.bufferTypeFromDtype(grid.local.dtype()); + const layout = platform.defaultMemoryLayout(dims, grid.local.dtype()); + + // The writers of each device's box; the first of them creates the + // allocation the others view. + var box_writers: [Platform.MAX_NUM_DEVICES]u64 = undefined; + for (grid.writers[0..grid.box_count]) |writers| { + var mask = writers; + while (mask != 0) : (mask &= mask - 1) box_writers[@ctz(mask)] = writers; + } + + // Creation holds a reference to every shared allocation and its + // views one each. Declared before the buffers' cleanup, so a failure + // destroys the views first and this frees the owners. + var shared: [Platform.MAX_NUM_DEVICES]?*SharedHostAllocation = @splat(null); + defer for (shared[0..shard_count]) |allocation| if (allocation) |a| a.release(); + var pjrt_buffers: Buffer.Shards = .empty; + errdefer for (pjrt_buffers.constSlice()) |buffer| buffer.deinit(api); + for (grid.devices, shards, 0..) |device, *shard, index| { + // The front end only submits host kinds when every device has + // a pinned host space (`backend.supportsMemory`). + const host = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; + if (grid.replicas() > 1 or destination.memory == .host_mapped) { + const first: usize = @ctz(box_writers[index]); + if (first == index) shared[index] = try .create(direct.io, platform, dims, element_type, layout, host); + const allocation = shared[first].?; + const space = platform.devices[device.id].memory(destination.memory.space()).?.pjrt_memory; + pjrt_buffers.appendAssumeCapacity(try allocation.view(platform, dims, element_type, layout, space)); + shard.* = allocation.data; + } else { + const buffer = try createHostBuffer(direct.io, platform, dims, element_type, layout, host); + pjrt_buffers.appendAssumeCapacity(buffer); + shard.* = @ptrCast(try buffer.opaqueDeviceMemoryDataPointer(api)); + } + } + + destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); + return .{ .host = shards }; + } + + fn deinit(self: *DestinationState, allocator: std.mem.Allocator, api: *const pjrt.Api) void { + for (self.targets) |target| target.manager.deinit(api); + allocator.free(self.targets); + allocator.free(self.host); + } + + /// Where a host transfer's bytes go: the allocation its writers, the + /// devices of one box, share. Writing it once writes every one of them. + fn hostLanding(self: *const DestinationState, transfer: Plan.Transfer) []u8 { + const first: usize = @ctz(transfer.writer_mask); + std.debug.assert(first < self.host.len); + return self.host[first][transfer.destination_offset..][0..transfer.len]; + } +}; + +/// One host allocation that the devices of a box read through views of it, +/// in their pinned host memory space or, for `host_mapped`, their default +/// one: the owner buffer is not a shard, every view holds a reference that +/// its deletion callback drops, and the last drop destroys the owner. PJRT +/// runs a view's callback once the view's memory is out of use, after any +/// execution still reading it, so the allocation outlives its readers +/// whatever order the shards are freed in, including views of the shards +/// (`Buffer.createView`) that their owner outlives. On the process heap: +/// the last view may go long after the loader. +/// Writing through one view shows on every device, so a shard of a +/// replicated host-resident destination must not be donated to an +/// execution that writes it, as with `Buffer.HostAccessible.init(.undonatable)`. +const SharedHostAllocation = struct { + /// Allocations not released yet, counted in tests only. + var live: if (builtin.is_test) std.atomic.Value(usize) else void = if (builtin.is_test) .init(0) else {}; + + api: *const pjrt.Api, + owner: *pjrt.Buffer, + data: [*]u8, + /// One per live view, and one for `DestinationState.initHost` while it + /// creates them. + references: std.atomic.Value(usize) = .init(1), + + fn create( + io: std.Io, + platform: *const Platform, + dims: []const i64, + element_type: pjrt.BufferType, + layout: pjrt.MemoryLayout, + memory: *const pjrt.Memory, + ) StateInitError!*SharedHostAllocation { + const api = platform.pjrt_api; + const self = try std.heap.smp_allocator.create(SharedHostAllocation); + errdefer std.heap.smp_allocator.destroy(self); + const owner = try createHostBuffer(io, platform, dims, element_type, layout, memory); + errdefer owner.deinit(api); + self.* = .{ + .api = api, + .owner = owner, + .data = @ptrCast(try owner.opaqueDeviceMemoryDataPointer(api)), + }; + if (builtin.is_test) _ = live.fetchAdd(1, .monotonic); + return self; + } + + /// A shard in `memory`, holding a reference: the host space of its + /// device, or the default one, where kernels read the allocation in + /// place. Every device can read it. + fn view( + self: *SharedHostAllocation, + platform: *const Platform, + dims: []const i64, + element_type: pjrt.BufferType, + layout: pjrt.MemoryLayout, + memory: *const pjrt.Memory, + ) StateInitError!*pjrt.Buffer { + _ = self.references.fetchAdd(1, .monotonic); + errdefer self.release(); + return platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ + .device_buffer_ptr = self.data, + .dims = dims, + .element_type = element_type, + .layout = layout, + .memory = memory, + .on_delete_callback = &onViewDeleted, + .on_delete_callback_arg = self, + }); + } + + fn onViewDeleted(_: ?*anyopaque, context: ?*anyopaque) callconv(.c) void { + const self: *SharedHostAllocation = @ptrCast(@alignCast(context.?)); + self.release(); + } + + fn release(self: *SharedHostAllocation) void { + if (self.references.fetchSub(1, .acq_rel) != 1) return; + self.owner.deinit(self.api); + std.heap.smp_allocator.destroy(self); + if (builtin.is_test) _ = live.fetchSub(1, .monotonic); + } +}; + +/// Shared host allocations not released yet, in tests: the front end's +/// tests count the ones their views keep. +pub fn liveHostAllocations() usize { + comptime std.debug.assert(builtin.is_test); + return SharedHostAllocation.live.load(.acquire); +} + +/// An uninitialized buffer in a host memory space, once its allocation +/// completed: its host pointer is valid only then. +fn createHostBuffer( + io: std.Io, + platform: *const Platform, + dims: []const i64, + element_type: pjrt.BufferType, + layout: pjrt.MemoryLayout, + memory: *const pjrt.Memory, +) StateInitError!*pjrt.Buffer { + const api = platform.pjrt_api; + const buffer = try platform.pjrt_client.createUninitializedBuffer(api, .{ + .dims = dims, + .element_type = element_type, + .layout = layout, + .dst = .{ .memory = memory }, + }); + errdefer buffer.deinit(api); + const ready = buffer.readyEvent(api); + defer ready.deinit(api); + try ready.await(api, io); + return buffer; +} + +/// The lifecycle credits: one per request the pre-grown pinned set holds, +/// each a fixed slot of `blocks_per_slot` blocks. A worker takes a credit +/// before claiming a job, and the request gives it back with its last +/// reference: after its last DMA callback, or at the end of a host job. Only +/// the holder reads into, lands in and transfers from its slot's blocks, so +/// the credits are the one bound on pinned memory in use. A block whose +/// pieces are done stays with its request until the request ends; nobody +/// else could use it anyway, since every credit owns a whole slot. +const Credits = struct { + /// Every slot's blocks, slot `s` at `blocks[s * blocks_per_slot ..]`; + /// a tail shorter than a slot is left unused. + blocks: []const host_memory.Block, + blocks_per_slot: usize, + /// Free slots, a stack of indices; owned by `mutex`. + free: []u32, + free_len: usize, + /// The most slots held at once. + high_water: usize = 0, + mutex: std.Io.Mutex = .init, + /// Waiters for a slot; one release wakes one of them. + condition: std.Io.Condition = .init, + + fn init(allocator: std.mem.Allocator, blocks: []const host_memory.Block, blocks_per_slot: usize) std.mem.Allocator.Error!Credits { + std.debug.assert(blocks_per_slot != 0); + const free = try allocator.alloc(u32, blocks.len / blocks_per_slot); + // Slot 0 first. + for (free, 0..) |*slot, index| slot.* = @intCast(free.len - 1 - index); + return .{ .blocks = blocks, .blocks_per_slot = blocks_per_slot, .free = free, .free_len = free.len }; + } + + fn deinit(self: *Credits, allocator: std.mem.Allocator) void { + std.debug.assert(self.free_len == self.free.len); + allocator.free(self.free); + } + + fn count(self: *const Credits) usize { + return self.free.len; + } + + /// Waits for a free slot. Requests in flight give theirs back even after + /// a failure, so a waiter always gets one. + fn acquire(self: *Credits, io: std.Io) u32 { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (self.free_len == 0) { + self.condition.waitUncancelable(io, &self.mutex); + } + self.free_len -= 1; + self.high_water = @max(self.high_water, self.free.len - self.free_len); + return self.free[self.free_len]; + } + + fn release(self: *Credits, io: std.Io, slot: u32) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + std.debug.assert(self.free_len < self.free.len); + self.free[self.free_len] = slot; + self.free_len += 1; + // One release frees one slot: waking every waiter here would be a + // thundering herd. + self.condition.signal(io); + } + + fn slotBlocks(self: *const Credits, slot: u32) []const host_memory.Block { + return self.blocks[slot * self.blocks_per_slot ..][0..self.blocks_per_slot]; + } + + fn inUse(self: *Credits, io: std.Io) usize { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + return self.free.len - self.free_len; + } +}; + +/// The debug summary's counters (`Loader.logSummary`): read only by logs; +/// nothing here decides. +const Metrics = struct { + read_operations: std.atomic.Value(u64) = .init(0), + read_bytes: std.atomic.Value(u64) = .init(0), + dma_submissions: std.atomic.Value(u64) = .init(0), + /// Time workers spend waiting for a lifecycle credit and inside the + /// source read, summed over requests: waits above the read time mean the + /// load is DMA-completion bound. + credit_wait_ns: std.atomic.Value(u64) = .init(0), + read_ns: std.atomic.Value(u64) = .init(0), + /// From a request's enqueue to its last DMA callback: how long the DMA + /// stage holds a lifecycle credit and pinned blocks per request. + dma_stage_ns: std.atomic.Value(u64) = .init(0), + /// Inside `submitTransfer` (the PJRT call and the callback registration) + /// and from a submission to its ready callback, summed over submissions. + dma_submit_ns: std.atomic.Value(u64) = .init(0), + dma_piece_ns: std.atomic.Value(u64) = .init(0), + /// Why a device pump stopped: nothing queued, or no room under the + /// device's budget. A pump that mostly stops empty is starved by the + /// reads; one that mostly stops full is waiting on the engine. + pump_stops_empty: std.atomic.Value(u64) = .init(0), + pump_stops_full: std.atomic.Value(u64) = .init(0), + /// Inside `ensureState` for a claimed job's destinations: the first + /// worker to touch a tensor creates its PJRT buffers and transfer + /// managers there, and the other workers of the same tensor wait for it; + /// a host job waits there for the allocation task. + tensor_init_ns: std.atomic.Value(u64) = .init(0), + /// The allocation task's time creating host-resident destinations ahead + /// of their reads, summed over batches. + ahead_init_ns: std.atomic.Value(u64) = .init(0), +}; + +/// A value initialized at most once by whichever task touches it first, +/// with the initializer that task passes. Concurrent callers wait on the +/// event; a failed initialization keeps its error code and re-materializes +/// the same error for every later caller. +fn LazyOnce(comptime T: type) type { + return struct { + const Self = @This(); + const Status = enum(u8) { + uninitialized, + initializing, + ready, + failed, + }; + + value: T = undefined, + status: std.atomic.Value(Status) = .init(.uninitialized), + error_code: std.atomic.Value(u16) = .init(0), + initialized: std.Io.Event = .unset, + + /// The error set of `initFn`, which returns `T`. + fn InitError(comptime initFn: anytype) type { + return @typeInfo(@typeInfo(@TypeOf(initFn)).@"fn".return_type.?).error_union.error_set; + } + + fn ensure(self: *Self, io: std.Io, comptime initFn: anytype, args: std.meta.ArgsTuple(@TypeOf(initFn))) InitError(initFn)!*T { + while (true) switch (self.status.load(.acquire)) { + .uninitialized => { + if (self.status.cmpxchgStrong(.uninitialized, .initializing, .acq_rel, .acquire) != null) continue; + self.value = @call(.auto, initFn, args) catch |err| { + self.error_code.store(@intFromError(err), .release); + self.status.store(.failed, .release); + self.initialized.set(io); + return err; + }; + self.status.store(.ready, .release); + self.initialized.set(io); + return &self.value; + }, + .initializing => self.initialized.waitUncancelable(io), + .ready => return &self.value, + .failed => return @errorCast(@errorFromInt(self.error_code.load(.acquire))), + }; + } + + /// The value when initialization has completed successfully. + fn readyValue(self: *Self) ?*T { + return if (self.status.load(.acquire) == .ready) &self.value else null; + } + }; +} + +fn secondsBetween(from: std.Io.Timestamp, to: std.Io.Timestamp) f64 { + return @as(f64, @floatFromInt(from.durationTo(to).nanoseconds)) / std.time.ns_per_s; +} + +/// Mean milliseconds per unit of a summed duration (zero units: the total). +fn millisecondsPer(total_ns: u64, units: u64) f64 { + return @as(f64, @floatFromInt(total_ns)) / std.time.ns_per_ms / @as(f64, @floatFromInt(@max(units, 1))); +} + +fn awakeNs(io: std.Io) u64 { + const now: std.Io.Timestamp = .now(io, .awake); + return @intCast(@max(now.nanoseconds, 1)); +} + +/// Writes `contents` to a new file `name` in `dir` and returns its real +/// path, in `path_buffer`. +fn writeTestFile(dir: std.Io.Dir, io: std.Io, name: []const u8, contents: []const u8, path_buffer: []u8) ![]const u8 { + const file = try dir.createFile(io, name, .{ .read = true }); + defer file.close(io); + try file.writePositionalAll(io, contents, 0); + return path_buffer[0..try file.realPath(io, path_buffer)]; +} + +// Planner tests. + +/// Every file read at `alignment`, none opened to ask. +fn fixedReads(alignment: usize) Planner.Reads { + return .{ .io = undefined, .vfs = null, .policy = .off, .alignment = alignment }; +} + +/// What a loader's `Planner.Reads` predicts for `checkpoint` under `policy`. +fn testPrediction(io: std.Io, policy: VFS.DirectIo, checkpoint: ?[]const u8) !Planner.Reads.Prediction { + var reads: Planner.Reads = .init(io, policy); + defer reads.deinit(std.testing.allocator); + return reads.predict(std.testing.allocator, checkpoint); +} + +/// `Planner.preparePlan` with every file read at `alignment`, kept in +/// `arena`; the testing allocator checks the cut for leaks. +fn prepareTestPlan(arena: std.mem.Allocator, parts: []Part, order: []const usize, config: Planner.Config, alignment: usize) !*Plan { + return Planner.preparePlan(std.testing.allocator, arena, parts, order, config, fixedReads(alignment)); +} + +/// One plain destination per source, as `createDestinations` builds them +/// for whole-tensor specs, without device state: for planner tests. +fn initPlainParts( + platform: *const Platform, + sources: []safetensors.Tensor, + slots: []const *SourceSlot, + outputs: []Buffer, + destinations: []Destination, + parts: []Part, +) void { + for (sources, slots, outputs, destinations, parts) |*source, slot, *output, *destination, *part| { + part.* = .{ + .source = source, + .source_slot = slot, + .destination = destination, + .lattice = .{ .width = source.byteSize() }, + }; + destination.* = .{ + .shape = source.shape, + .grid = .init(platform, source.shape), + .output = output, + .parts = @as(*[1]Part, part), + .unread_bytes = .init(source.byteSize()), + }; + } +} + +/// Claims out and frees an open batch that a test only inspected; it must +/// be the only queued batch. +fn discardTestBatch(scheduler: *Scheduler, batch: *Batch) void { + const io = std.testing.io; + var claimed: usize = 0; + while (scheduler.claim(io)) |claim| { + std.debug.assert(claim.plan.batch == batch); + claimed += 1; + } + batch.releaseUnits(1 + claimed); + std.debug.assert(batch.done.isSet()); + batch.destroy(); +} + +test "the planner opens only local files to ask about direct I/O" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents: [2 * 4096]u8 = @splat(7); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "drafter.bin", &contents, &path_buffer); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + // A scheme whose files the plain io cannot open: asking would fail. + try vfs.registerBackend("remote", .{ .io = io }); + const vfs_io = vfs.io(); + const reads: Planner.Reads = .{ .io = vfs_io, .vfs = &vfs, .policy = .on, .alignment = 4096 }; + try std.testing.expectEqual(@as(usize, 0), reads.boundOf("remote://bucket/model.safetensors")); + try std.testing.expectEqual(@as(usize, 4096), reads.boundOf(path)); + + // A remote file is read exact, without an open on the planner. + var remote: SourceSlot = .{ .uri = "remote://bucket/model.safetensors" }; + defer remote.deinit(vfs_io); + try std.testing.expectEqual(@as(usize, 0), try reads.alignmentOf(&remote)); + try std.testing.expect(remote.file.readyValue() == null); + + // A local file beside it is opened and asked, whatever the checkpoint: + // direct where the filesystem allows it. + var local: SourceSlot = .{ .uri = path }; + defer local.deinit(vfs_io); + const alignment = try reads.alignmentOf(&local); + const file = local.file.readyValue().?.*; + try std.testing.expectEqual(@as(usize, if (vfs.useDirectIo(file, .on)) 4096 else 0), alignment); +} + +test "the checkpoint's reads are predicted as the VFS will decide the files its registry reads" { + if (VFS.direct_io_alignment == 0) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const long: [2 * VFS.direct_io_alignment + 100]u8 = @splat(7); + try tmp.dir.writeFile(io, .{ .sub_path = "a.safetensors", .data = &long }); + try tmp.dir.writeFile(io, .{ .sub_path = "b.safetensors", .data = &long }); + try tmp.dir.writeFile(io, .{ .sub_path = "c.safetensors", .data = long[0..100] }); + // The index lists a, b, c and a symlink to a; the registry does not + // read the consolidated file beside them (Mistral's format). The files + // hold no header: none is read. + try tmp.dir.symLink(io, "a.safetensors", "alias.safetensors", .{}); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"x\":\"a.safetensors\",\"y\":\"b.safetensors\",\"z\":\"c.safetensors\",\"w\":\"alias.safetensors\"}}" }); + try tmp.dir.writeFile(io, .{ .sub_path = "consolidated.safetensors", .data = &long }); + // A snapshot whose file links into blobs, as a Hugging Face cache's do. + const blob: [8 * 1024]u8 = @splat(9); + try tmp.dir.createDir(io, "blobs", .default_dir); + try tmp.dir.writeFile(io, .{ .sub_path = "blobs/x", .data = &blob }); + try tmp.dir.createDir(io, "snapshot", .default_dir); + try tmp.dir.symLink(io, "../blobs/x", "snapshot/model.safetensors", .{}); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; + var path_buffer: [std.fs.max_path_bytes + 64]u8 = undefined; + + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + // A remote scheme: nothing is predicted for a checkpoint there. + try vfs.registerBackend("remote", .{ .io = io }); + const vfs_io = vfs.io(); + + // The listed files as fresh handles decide them under `on`: c, shorter + // than a unit, never direct. + var expected: Planner.Reads.Prediction = .{}; + const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); + defer dir.close(vfs_io); + for ([_][]const u8{ "a.safetensors", "b.safetensors", "c.safetensors" }) |name| { + const file = try dir.openFile(vfs_io, name, .{}); + defer file.close(vfs_io); + const size = (try file.stat(vfs_io)).size; + expected.files += 1; + expected.bytes += size; + if (vfs.useDirectIo(file, .on)) expected.direct_bytes += size; + } + try std.testing.expectEqual(@as(u64, 2 * long.len + 100), expected.bytes); + + var on_reads: Planner.Reads = .init(vfs_io, .on); + defer on_reads.deinit(allocator); + // The symlink is predicted once, with a. + try std.testing.expectEqual(expected, try on_reads.predict(allocator, root)); + // Each answer is kept by the file's real path, none for the + // consolidated file. Under `on` each keeps `on`, c too: planning + // decides it, and it reads buffered. + try std.testing.expectEqual(@as(usize, 3), on_reads.predicted.count()); + try std.testing.expectEqual(VFS.DirectIo.on, on_reads.predicted.get(try std.fmt.bufPrint(&path_buffer, "{s}/c.safetensors", .{root})).?); + try std.testing.expectEqual(expected, try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/model.safetensors.index.json", .{root}))); + const one = try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/a.safetensors", .{root})); + try std.testing.expectEqual(@as(usize, 1), one.files); + try std.testing.expectEqual(@as(u64, long.len), one.bytes); + const snapshot = try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/snapshot", .{root})); + try std.testing.expectEqual(@as(usize, 1), snapshot.files); + try std.testing.expectEqual(@as(u64, blob.len), snapshot.bytes); + + // Just written, so cached: none reads direct under `auto`, and each is + // kept as buffered. + var auto_reads: Planner.Reads = .init(vfs_io, .auto); + defer auto_reads.deinit(allocator); + const cached = try auto_reads.predict(allocator, root); + try std.testing.expectEqual(@as(usize, 3), cached.files); + try std.testing.expectEqual(@as(u64, 0), cached.direct_bytes); + var answers = auto_reads.predicted.valueIterator(); + while (answers.next()) |answer| try std.testing.expectEqual(VFS.DirectIo.off, answer.*); + + // Nothing to predict: no direct I/O, no VFS, a remote or no checkpoint, + // or none there. + const nothing = [_]Planner.Reads.Prediction{ + try testPrediction(vfs_io, .off, root), + try testPrediction(io, .on, root), + try testPrediction(vfs_io, .on, "remote://bucket/model"), + try testPrediction(vfs_io, .on, null), + try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/missing", .{root})), + try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/blobs", .{root})), + }; + for (nothing) |prediction| try std.testing.expectEqual(@as(usize, 0), prediction.files); +} + +test "a file predicted to read direct is planned direct, though the prediction's sample cached it" { + if (comptime VFS.direct_io_alignment == 0) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // A snapshot whose file links into blobs, as a Hugging Face cache's + // does. 64 KiB: every page of it is sampled, on 4 KiB pages too. + const data: [64 * 1024]u8 = @splat(7); + const blobs = try tmp.dir.createDirPathOpen(io, "blobs", .{}); + defer blobs.close(io); + try testing.writeSafetensorsFile(allocator, io, blobs, "x", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{data.len}, .bytes = &data }, + }); + try tmp.dir.createDir(io, "snapshot", .default_dir); + try tmp.dir.symLink(io, "../blobs/x", "snapshot/model.safetensors", .{}); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; + var path_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + const checkpoint = try std.fmt.bufPrint(&path_buffer, "{s}/snapshot", .{root}); + + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + const vfs_io = vfs.io(); + // The planner's slot takes the registry's name for the file: the blob. + var registry: safetensors.TensorRegistry = try .fromPath(allocator, vfs_io, checkpoint); + defer registry.deinit(); + const uri = registry.tensors.get("w").?.file_uri; + { + const file = try blobs.openFile(io, "x", .{}); + defer file.close(io); + try file.sync(io); + _ = std.os.linux.fadvise(file.handle, 0, 0, std.os.linux.POSIX_FADV.DONTNEED); + } + + var reads: Planner.Reads = .init(vfs_io, .auto); + defer reads.deinit(allocator); + const predicted = try reads.predict(allocator, checkpoint); + // A filesystem that kept the file cached (tmpfs) or refuses direct + // reads says nothing here. + if (predicted.direct_bytes == 0) return error.SkipZigTest; + // Once the sample's reads landed, a probe of its own finds the file + // cached (a kernel that queues no read for a missed sample shows + // nothing). + try io.sleep(.fromMilliseconds(250), .awake); + { + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, uri, .{ .mode = .read_only }); + defer file.close(vfs_io); + if (vfs.useDirectIo(file, .auto)) return error.SkipZigTest; + } + + // Planning reads it as predicted: direct, the handle decided so. + var slot: SourceSlot = .{ .uri = uri }; + defer slot.deinit(vfs_io); + try std.testing.expectEqual(reads.alignment, try reads.alignmentOf(&slot)); + try std.testing.expect(vfs.useDirectIo(slot.file.readyValue().?.*, .auto)); +} + +test "arenas are placed for direct reads from four fifths of the checkpoint's bytes" { + const Case = struct { direct_bytes: u64, bytes: u64, direct: bool }; + const cases = [_]Case{ + .{ .direct_bytes = 0, .bytes = 0, .direct = false }, + .{ .direct_bytes = 4, .bytes = 5, .direct = true }, + .{ .direct_bytes = 79, .bytes = 100, .direct = false }, + .{ .direct_bytes = 80, .bytes = 100, .direct = true }, + .{ .direct_bytes = 0, .bytes = 100, .direct = false }, + }; + for (cases) |case| { + const prediction: Planner.Reads.Prediction = .{ .bytes = case.bytes, .direct_bytes = case.direct_bytes }; + try std.testing.expectEqual(case.direct, prediction.direct()); + } +} + +test "a strided part whose source runs are not back to back counts its whole source range" { + // [32, 16] u8 at a file offset off a 64-byte alignment. + const source: safetensors.Tensor = .{ .file_uri = "wide.bin", .name = "wide", .shape = .init(.{ 32, 16 }, .u8), .offset = 100 }; + const config: Planner.Config = .{ .block_size = 256, .request_size = 512 }; + // Transposed, 512 runs of one byte: every job reads all 512 bytes, + // 576 widened to the alignment. + const transposed: LoadPart = .{ .source = &source, .lattice = .{ .width = 1, .dims = &.{ + .{ .count = 16, .src_stride = 1, .dst_stride = 32 }, + .{ .count = 32, .src_stride = 16, .dst_stride = 1 }, + } } }; + try std.testing.expectEqual(@as(backend.Backend.StridedRead, .{ .read = 512, .limit = 512 }), Planner.stridedRead(config, fixedReads(0), transposed)); + try std.testing.expectEqual(576, Planner.stridedRead(config, fixedReads(64), transposed).read); + // A bigger block calibration picked does not raise the limit past the + // smallest one's request. + const bigger_block: Planner.Config = .{ .block_size = 1024, .request_size = 1024, .gather_limit = 512 }; + try std.testing.expectEqual(512, Planner.stridedRead(bigger_block, fixedReads(0), transposed).limit); + // Every other row, in order: the gaps stretch a job's range, so the + // whole range counts, from the first row to the last. + const every_other_row: LoadPart = .{ .source = &source, .lattice = .{ .width = 16, .dims = &.{.{ .count = 16, .src_stride = 32, .dst_stride = 16 }} } }; + try std.testing.expectEqual(15 * 32 + 16, Planner.stridedRead(config, fixedReads(0), every_other_row).read); + // Rows back to back read within their job's range. + const rows: LoadPart = .{ .source = &source, .lattice = .{ .width = 16, .dims = &.{.{ .count = 32, .src_stride = 16, .dst_stride = 16 }} } }; + try std.testing.expectEqual(0, Planner.stridedRead(config, fixedReads(64), rows).read); +} + +test "source planner coalesces exact adjacent and overlapping tensor ranges per file" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "a", .name = "a0", .shape = .init(.{4}, .u8), .offset = 10 }, + .{ .file_uri = "a", .name = "a1", .shape = .init(.{4}, .u8), .offset = 14 }, + .{ .file_uri = "a", .name = "a0-copy", .shape = .init(.{4}, .u8), .offset = 10 }, + .{ .file_uri = "a", .name = "a-gap", .shape = .init(.{4}, .u8), .offset = 20 }, + .{ .file_uri = "b", .name = "b0", .shape = .init(.{12}, .u8), .offset = 3 }, + }; + var slots = [_]SourceSlot{ + .{ .uri = "a" }, + .{ .uri = "b" }, + }; + var outputs: [sources.len]Buffer = undefined; + var destinations: [sources.len]Destination = undefined; + var parts: [sources.len]Part = undefined; + initPlainParts(platform, &sources, &.{ &slots[0], &slots[0], &slots[0], &slots[0], &slots[1] }, &outputs, &destinations, &parts); + + var scheduler: Scheduler = .init(allocator); + defer scheduler.deinit(); + const batch = try Batch.create(allocator, io, 0); + try Planner.publishFiles(&scheduler, io, batch, &parts, .{ + .block_size = 4, + .request_size = 8, + }, fixedReads(0)); + + // One plan per file. a:[10,18) merges adjacency and the duplicate, + // a:[20,24) remains exact, and b:[3,15) is split at the request-size + // boundary. + const plans = batch.plans.items; + try std.testing.expectEqual(@as(usize, 2), plans.len); + try std.testing.expect(plans[0].source_slot == &slots[0]); + try std.testing.expect(plans[1].source_slot == &slots[1]); + try std.testing.expectEqual(@as(usize, 2), plans[0].jobs.len); + try std.testing.expectEqual(@as(usize, 4), plans[0].transfers.len); + try std.testing.expectEqual(@as(u64, 10), plans[0].jobs[0].file_offset); + try std.testing.expectEqual(@as(usize, 8), plans[0].jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 3), plans[0].jobs[0].transfer_len); + try std.testing.expectEqual(@as(u64, 20), plans[0].jobs[1].file_offset); + try std.testing.expectEqual(@as(usize, 2), plans[1].jobs.len); + try std.testing.expectEqual(@as(usize, 3), plans[1].transfers.len); + try std.testing.expectEqual(@as(u64, 3), plans[1].jobs[0].file_offset); + try std.testing.expectEqual(@as(usize, 8), plans[1].jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 4), plans[1].jobs[1].kind.dma_read.len); + try std.testing.expectEqual(@as(u64, 12), plans[0].source_bytes); + try std.testing.expectEqual(@as(u64, 12), plans[1].source_bytes); + // One writer per transfer on one device: one event per transfer. Jobs + // span their 4-byte blocks. + try std.testing.expectEqual(@as(usize, 4), plans[0].events.len); + try std.testing.expectEqual(@as(usize, 3), plans[1].events.len); + try std.testing.expectEqual(@as(usize, 2), plans[0].requests.len); + try std.testing.expectEqual(@as(usize, 2), plans[0].jobs[0].blockLen(4)); + try std.testing.expectEqual(@as(usize, 1), plans[0].jobs[1].blockLen(4)); + try std.testing.expectEqual(@as(usize, 2), plans[1].jobs[0].blockLen(4)); + try std.testing.expectEqual(@as(usize, 1), plans[1].jobs[1].blockLen(4)); + for (plans) |plan| { + for (plan.requests) |*request| try std.testing.expectEqual(@as(usize, 0), request.pending.load(.acquire)); + } + discardTestBatch(&scheduler, batch); + + var aligned_sources = [_]safetensors.Tensor{ + .{ .file_uri = "aligned", .name = "aligned0", .shape = .init(.{7}, .u8), .offset = 0 }, + .{ .file_uri = "aligned", .name = "aligned1", .shape = .init(.{8}, .u8), .offset = 7 }, + .{ .file_uri = "aligned", .name = "aligned2", .shape = .init(.{5}, .u8), .offset = 15 }, + }; + var aligned_slot: SourceSlot = .{ .uri = "aligned" }; + var aligned_outputs: [aligned_sources.len]Buffer = undefined; + var aligned_destinations: [aligned_sources.len]Destination = undefined; + var aligned_items: [aligned_sources.len]Part = undefined; + initPlainParts(platform, &aligned_sources, &.{ &aligned_slot, &aligned_slot, &aligned_slot }, &aligned_outputs, &aligned_destinations, &aligned_items); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const aligned_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ + .block_size = 4, + .request_size = 8, + }, 0); + try std.testing.expectEqual(@as(usize, 3), aligned_plan.jobs.len); + try std.testing.expectEqual(@as(usize, 6), aligned_plan.transfers.len); + try std.testing.expectEqual(@as(usize, 7), aligned_plan.jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 8), aligned_plan.jobs[1].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 5), aligned_plan.jobs[2].kind.dma_read.len); + + // Widened for direct I/O: the same three jobs (16 - 2 * 4 = 8 still cuts + // at the tensor-safe boundaries 7 and 15), each read at 4-byte bounds, + // its transfers relative to the widened start; the last read runs past + // the tensor data, of which every byte is required. + const widened_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ + .block_size = 4, + .request_size = 16, + }, 4); + try std.testing.expectEqual(@as(usize, 3), widened_plan.jobs.len); + try std.testing.expectEqual(@as(u64, 0), widened_plan.jobs[0].file_offset); + try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 7), widened_plan.jobs[0].kind.dma_read.minimum_len); + try std.testing.expectEqual(@as(u64, 4), widened_plan.jobs[1].file_offset); + try std.testing.expectEqual(@as(usize, 12), widened_plan.jobs[1].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 11), widened_plan.jobs[1].kind.dma_read.minimum_len); + try std.testing.expectEqual(@as(usize, 3), widened_plan.jobs[1].blockLen(4)); + try std.testing.expectEqual(@as(usize, 3), widened_plan.transfers[widened_plan.jobs[1].transfer_start].landing); + try std.testing.expectEqual(@as(u64, 12), widened_plan.jobs[2].file_offset); + try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[2].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[2].kind.dma_read.minimum_len); + try std.testing.expectEqual(@as(usize, 3), widened_plan.transfers[widened_plan.jobs[2].transfer_start].landing); + try std.testing.expectEqual(@as(u64, 20), widened_plan.source_bytes); +} + +test "host-resident and DMA parts never share a job, and overlapping host parts read through blocks" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 2 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + // x, y and z end to end in one file. x lands in two host destinations, + // y in device memory between them, z in a third host destination. + const shape = Shape.init(.{ 8, 8 }, .u8); + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "file", .name = "x", .shape = shape, .offset = 0 }, + .{ .file_uri = "file", .name = "y", .shape = shape, .offset = 64 }, + .{ .file_uri = "file", .name = "z", .shape = shape, .offset = 128 }, + }; + var slot: SourceSlot = .{ .uri = "file" }; + var outputs: [4]Buffer = undefined; + var destinations: [4]Destination = undefined; + var parts = [_]Part{ + .{ .source = &sources[0], .source_slot = &slot, .destination = &destinations[0], .lattice = .{ .width = 64 } }, + .{ .source = &sources[1], .source_slot = &slot, .destination = &destinations[1], .lattice = .{ .width = 64 } }, + .{ .source = &sources[0], .source_slot = &slot, .destination = &destinations[2], .lattice = .{ .width = 64 } }, + .{ .source = &sources[2], .source_slot = &slot, .destination = &destinations[3], .lattice = .{ .width = 64 } }, + }; + const memories = [_]Memory{ .host_pinned, .default, .host_pinned, .host_pinned }; + for (&destinations, &outputs, &parts, memories) |*destination, *output, *part, memory| { + destination.* = .{ .shape = shape, .grid = .init(platform, shape), .memory = memory, .output = output, .parts = @as(*[1]Part, part), .unread_bytes = .init(64) }; + } + + const order = try Planner.sortedPartOrder(allocator, &parts); + defer allocator.free(order); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const plan = try prepareTestPlan(arena.allocator(), &parts, order, .{ + .block_size = 64, + .request_size = 1024, + }, 0); + // x's run reads its bytes once for two destinations, so through + // blocks; y goes out by DMA; z is read in place. File order. + const expected = [_]struct { offset: u64, kind: std.meta.Tag(Plan.Job.Kind) }{ + .{ .offset = 0, .kind = .host_bounce }, + .{ .offset = 64, .kind = .dma_read }, + .{ .offset = 128, .kind = .host_direct }, + }; + try std.testing.expectEqual(expected.len, plan.jobs.len); + var dma_writers: usize = 0; + for (plan.jobs, expected) |job, want| { + try std.testing.expectEqual(want.offset, job.file_offset); + try std.testing.expectEqual(want.kind, std.meta.activeTag(job.kind)); + for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| { + try std.testing.expectEqual(job.kind != .dma_read, transfer.destination.memory.isHost()); + if (job.kind == .dma_read) dma_writers += @popCount(transfer.writer_mask); + } + } + try std.testing.expectEqual(@as(usize, 2), plan.jobs[0].transfer_len); + try std.testing.expectEqual(@as(usize, 1), plan.jobs[0].blockLen(64)); + try std.testing.expectEqual(dma_writers, plan.events.len); +} + +/// Plans a composite destination fed by three parts from two files, with a +/// gap and a tail of zero fill, beside a plain one, then replays the plans on +/// host memory: every device's placement bytes are written exactly once, +/// from the right source bytes or from zeros. +const CompositeTest = struct { + const Options = struct { + block_size: usize, + request_size: usize, + memory: Memory = .default, + alignment: usize = 0, + /// By default nothing is staged. + staging_gain: usize = std.math.maxInt(usize), + /// Whether some piece holds a's last byte and b's first one. + cross_part_piece: bool, + }; + + /// With a host memory kind the destinations take host jobs: direct + /// without `alignment`, through its slot's blocks with it. A DMA job is + /// staged where it saves `staging_gain` submissions; the replay reads + /// its blocks from the two files as its worker would (`jobBlocks`). + fn run(platform: *const Platform, shape: Shape, options: Options) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const block_size = options.block_size; + var first_bytes: [96]u8 = undefined; + for (&first_bytes, 0..) |*byte, i| byte.* = @as(u8, @truncate(i *% 31 +% 7)) | 1; + var second_bytes: [128]u8 = undefined; + for (&second_bytes, 0..) |*byte, i| byte.* = @as(u8, @truncate(i *% 17 +% 3)) | 0x80; + // Rows of 16 bytes. a and b are adjacent in "first" and in the + // destination; c leaves a five-row gap before it and a two-row tail. + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "first", .name = "a", .shape = .init(.{ 3, 8 }, .u16), .offset = 4 }, + .{ .file_uri = "first", .name = "b", .shape = .init(.{ 2, 8 }, .u16), .offset = 52 }, + .{ .file_uri = "second", .name = "c", .shape = .init(.{ 4, 8 }, .u16), .offset = 0 }, + .{ .file_uri = "second", .name = "plain", .shape = .init(.{ 4, 8 }, .u16), .offset = 64 }, + }; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var paths: [2][std.Io.Dir.max_path_bytes]u8 = undefined; + var slots: [2]SourceSlot = undefined; + var opened: usize = 0; + defer for (slots[0..opened]) |*slot| slot.deinit(io); + for (&slots, &paths, [_][]const u8{ "first", "second" }, [_][]const u8{ &first_bytes, &second_bytes }) |*slot, *path, name, bytes| { + slot.* = .{ .uri = try writeTestFile(tmp.dir, io, name, bytes, path) }; + opened += 1; + } + var fixture: TestLoader = undefined; + try fixture.init(1); + defer fixture.deinit(); + const loader = &fixture.loader; + loader.plan_config = .{ .block_size = block_size, .request_size = options.request_size }; + var outputs: [2]Buffer = undefined; + var destinations: [2]Destination = undefined; + var parts = [_]Part{ + .{ .source = &sources[0], .source_slot = &slots[0], .destination = &destinations[0], .lattice = .{ .dst_offset = 0, .width = 48 } }, + .{ .source = &sources[1], .source_slot = &slots[0], .destination = &destinations[0], .lattice = .{ .dst_offset = 48, .width = 32 } }, + .{ .source = &sources[2], .source_slot = &slots[1], .destination = &destinations[0], .lattice = .{ .dst_offset = 160, .width = 64 } }, + .{ .source = &sources[3], .source_slot = &slots[1], .destination = &destinations[1], .lattice = .{ .width = 64 } }, + }; + const plain_shape = shape.setDim(0, 4); + destinations = .{ + .{ .shape = shape, .grid = .init(platform, shape), .memory = options.memory, .output = &outputs[0], .parts = parts[0..3], .unread_bytes = .init(144) }, + .{ .shape = plain_shape, .grid = .init(platform, plain_shape), .memory = options.memory, .output = &outputs[1], .parts = parts[3..4], .unread_bytes = .init(64) }, + }; + const host = destinations[0].memory.isHost(); + + const order = try Planner.sortedPartOrder(allocator, &parts); + defer allocator.free(order); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + var plans: [2]*Plan = undefined; + var plan_count: usize = 0; + var start: usize = 0; + while (start < order.len) : (plan_count += 1) { + const end = Planner.fileGroupEnd(&parts, order, start); + plans[plan_count] = try prepareTestPlan(arena.allocator(), &parts, order[start..end], .{ + .block_size = block_size, + .request_size = options.request_size, + .minimum_staging_gain = options.staging_gain, + }, options.alignment); + start = end; + } + try std.testing.expectEqual(@as(usize, 2), plan_count); + // The fill goes out once, with the plan of the destination's first + // part, as its last job. + const fill_kind: std.meta.Tag(Plan.Job.Kind) = if (host) .host_fill else .dma_fill; + try std.testing.expectEqual(fill_kind, std.meta.activeTag(plans[0].jobs[plans[0].jobs.len - 1].kind)); + // Host jobs take no DMA event; DMA jobs one event per writer. + for (plans[0..plan_count], [_]usize{ 1, 0 }) |plan, fill_jobs| { + var writers: usize = 0; + var fills: usize = 0; + for (plan.jobs) |job| { + const kind = std.meta.activeTag(job.kind); + if (kind == fill_kind) fills += 1; + if (host) { + const read_kind: std.meta.Tag(Plan.Job.Kind) = if (options.alignment == 0) .host_direct else .host_bounce; + try std.testing.expect(kind == .host_fill or kind == read_kind); + } else { + try std.testing.expect(kind == .dma_read or kind == .dma_land or kind == .dma_fill); + } + for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| writers += @popCount(transfer.writer_mask); + } + try std.testing.expectEqual(fill_jobs, fills); + try std.testing.expectEqual(if (host) 0 else writers, plan.events.len); + } + + const zeros = try allocator.alloc(u8, 256); + defer allocator.free(zeros); + @memset(zeros, 0); + var replays: [2]Replay = undefined; + var initialized: usize = 0; + defer for (replays[0..initialized]) |*replay| replay.deinit(allocator); + for (&replays, &destinations) |*replay, *destination| { + replay.* = try .init(allocator, platform.placedShape(destination.shape).packedShape()); + initialized += 1; + } + var cross_part_piece = false; + for (plans[0..plan_count]) |plan| { + const file_bytes: []const u8 = if (plan.source_slot == &slots[0]) &first_bytes else &second_bytes; + for (plan.jobs) |job| { + const fill = job.kind == .dma_fill or job.kind == .host_fill; + const blocks = if (host or fill) &.{} else try jobBlocks(allocator, loader, plan, job); + defer allocator.free(blocks); + for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| { + // Every transfer stays inside one block (`Plan.Transfer`). + try std.testing.expect(transfer.landing % block_size + transfer.len <= block_size); + switch (job.kind) { + .host_direct, .host_bounce => |read| try std.testing.expect(transfer.landing + transfer.len <= read.len), + else => try std.testing.expect(transfer.landing < (if (fill) 1 else job.blockLen(block_size)) * block_size), + } + const bytes = if (fill) fill: { + try std.testing.expectEqual(@as(usize, 0), transfer.landing); + break :fill zeros[0..transfer.len]; + } else if (host) read: { + // A host job's landings address its read, which + // starts at the job's file offset. + const file_offset = job.file_offset + transfer.landing; + if (plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; + break :read file_bytes[file_offset..][0..transfer.len]; + } else read: { + // a's last byte and b's first one in the same piece, + // where a read's blocks hold the file in order. + const file_offset = job.file_offset + transfer.landing; + if (job.kind == .dma_read and plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; + break :read blocks[transfer.landing..][0..transfer.len]; + }; + const index: usize = if (transfer.destination == &destinations[0]) 0 else 1; + try replays[index].write(transfer.writer_mask, transfer.destination_offset, bytes); + } + } + } + try std.testing.expectEqual(options.cross_part_piece, cross_part_piece); + + var global: [256]u8 = @splat(0); + @memcpy(global[0..80], first_bytes[4..84]); + @memcpy(global[160..224], second_bytes[0..64]); + try replays[0].expect(allocator, &global); + try replays[1].expect(allocator, second_bytes[64..128]); + } + + /// A DMA job's blocks taken end to end once its worker has read it: + /// the read in file order, or as its worker lands a landing job + /// (`landJob`). + fn jobBlocks(allocator: std.mem.Allocator, loader: *Loader, plan: *const Plan, job: Plan.Job) ![]u8 { + const block_size = loader.plan_config.block_size; + const bytes = try allocator.alloc(u8, job.blockLen(block_size) * block_size); + errdefer allocator.free(bytes); + @memset(bytes, 0xaa); + const land = switch (job.kind) { + .dma_read => |read| { + // The end of the file cuts a widened read's padding. + var buffers = [_][]u8{bytes[0..read.len]}; + try readSlot(loader, plan.source_slot, &buffers, job.file_offset, read.minimum_len); + return bytes; + }, + .dma_land => |land| land, + .dma_fill, .host_direct, .host_bounce, .host_fill => unreachable, + }; + // A bounce buffer as large as any of the job's reads. + var longest: usize = land.len; + for (plan.extents[land.extent_start..][0..land.extent_len]) |extent| longest = @max(longest, extent.len); + const blocks = try allocator.alloc(host_memory.Block, job.blockLen(block_size)); + defer allocator.free(blocks); + for (blocks, 0..) |*block, index| block.* = bytes[index * block_size ..][0..block_size]; + var scratch: ReadRequest.Scratch = try .init(allocator, Planner.blockCount(longest, block_size), block_size); + defer scratch.deinit(); + try landJob(loader, plan, land, blocks, &scratch); + return bytes; + } + + /// One destination's device buffers, written by replayed transfers. + const Replay = struct { + shape: Shape, + placement: Sharding.Placement, + writer_size: usize, + bytes: []u8, + written: []bool, + + fn init(allocator: std.mem.Allocator, placed_shape: Shape) !Replay { + const placement: Sharding.Placement = try .init(&placed_shape); + const writers = placement.mesh().?.devicesInCanonicalOrder().len; + const writer_size = placement.localShape(placed_shape).byteSize(); + const bytes = try allocator.alloc(u8, writers * writer_size); + errdefer allocator.free(bytes); + @memset(bytes, 0xaa); + const written = try allocator.alloc(bool, bytes.len); + @memset(written, false); + return .{ .shape = placed_shape, .placement = placement, .writer_size = writer_size, .bytes = bytes, .written = written }; + } + + fn deinit(self: *Replay, allocator: std.mem.Allocator) void { + allocator.free(self.written); + allocator.free(self.bytes); + } + + /// Each byte of each device at most once. + fn write(self: *Replay, writer_mask: u64, destination_offset: usize, source: []const u8) !void { + var mask = writer_mask; + while (mask != 0) { + const writer: usize = @intCast(@ctz(mask)); + mask &= mask - 1; + try std.testing.expect(destination_offset + source.len <= self.writer_size); + const at = writer * self.writer_size + destination_offset; + for (self.written[at..][0..source.len]) |*flag| { + try std.testing.expect(!flag.*); + flag.* = true; + } + @memcpy(self.bytes[at..][0..source.len], source); + } + } + + /// Every device holds its slice of the rank-2 `global`, entirely. + fn expect(self: *const Replay, allocator: std.mem.Allocator, global: []const u8) !void { + for (self.written) |flag| try std.testing.expect(flag); + const shape = self.shape; + const element_bytes = shape.dtype().sizeOf(); + const row_bytes: usize = @intCast(shape.dim(1) * element_bytes); + const expected = try allocator.alloc(u8, self.writer_size); + defer allocator.free(expected); + for (0..self.placement.mesh().?.devicesInCanonicalOrder().len) |writer| { + const slices = self.placement.slices(writer, shape); + const rows = slices.get(0); + const columns = slices.get(1); + const len: usize = @intCast(columns.size * element_bytes); + var cursor: usize = 0; + for (@intCast(rows.start)..@intCast(rows.start + rows.size)) |row| { + @memcpy(expected[cursor..][0..len], global[row * row_bytes + @as(usize, @intCast(columns.start * element_bytes)) ..][0..len]); + cursor += len; + } + try std.testing.expectEqualSlices(u8, expected, self.bytes[writer * self.writer_size ..][0..self.writer_size]); + } + } + }; +}; + +test "composite destinations place each part and zero-fill the rest of every device" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const shape = Shape.init(.{ .row = 16, .col = 8 }, .u16); + // Two by two devices, then two by four. + for ([_]u32{ 4, 8 }) |device_count| { + var platform = Platform.init(allocator, io, .cpu, .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = device_count }, + }) catch return error.SkipZigTest; + defer platform.deinit(allocator, io); + const physical = &platform.physical_mesh; + const logical: Sharding.LogicalMesh = .mesh(.{ .model = .high_bandwidth }); + const halves: Sharding.Mesh = try .init("halves", physical, logical, .parseBindings(.{ .model = .link_x })); + const quarters: Sharding.Mesh = if (device_count == 4) + try .init("quarters", physical, logical, .parseBindings(.{ .model = .{ .link_x, .link_y } })) + else + try .init("quarters", physical, logical, .parseBindings(.{ .model = .link_y })); + const all: Sharding.Mesh = try .init("all", physical, logical, .parseBindings(.{ .model = .{ .link_x, .link_y } })); + const scenarios = .{ + // Two row shards, each on half the devices: b and c straddle no + // boundary, the gap does. + .{ &halves, .{ .row = .model }, true }, + // Four row shards: b, c and the gap straddle shards. + .{ &quarters, .{ .row = .model }, true }, + // Every row of every source split across two devices, then + // across every device. + .{ &halves, .{ .col = .model }, false }, + .{ &all, .{ .col = .model }, false }, + // Every byte on every device: one span, so a and b merge. + .{ &all, .{}, true }, + }; + inline for (scenarios) |scenario| { + const placed = shape.withPartitioning(scenario[0], scenario[1]); + // 64-byte blocks hold a's tail and b's head together; 16-byte ones + // cut every row apart and every fill piece to one block. + try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .cross_part_piece = scenario[2] }); + try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 32, .cross_part_piece = false }); + // Staged wherever it saves a submission: the column-sharded jobs + // land grouped by device, in reads exact or widened. + for ([_]usize{ 0, 8 }) |alignment| { + try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .alignment = alignment, .staging_gain = 1, .cross_part_piece = scenario[2] }); + try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 48, .alignment = alignment, .staging_gain = 1, .cross_part_piece = false }); + } + // Host jobs, read in place or through blocks when the reads are + // widened, are cut by blocks the same way. + for ([_]usize{ 0, 8 }) |alignment| { + try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .memory = .host_pinned, .alignment = alignment, .cross_part_piece = scenario[2] }); + try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 32, .memory = .host_pinned, .alignment = alignment, .cross_part_piece = false }); + } + } + } +} + +// Scheduler tests. + +/// A plan of `job_count` unit jobs (`file_offset` = index), each with one +/// block and one event slot, and no transfers, in `arena`. +fn testPlan(arena: std.mem.Allocator, job_count: usize) !*Plan { + const jobs = try arena.alloc(Plan.Job, job_count); + const requests = try arena.alloc(ReadRequest, job_count); + @memset(requests, ReadRequest.idle); + const events = try arena.alloc(Dma.EventContext, job_count); + for (jobs, 0..) |*job, index| job.* = .{ + .file_offset = index, + .transfer_start = 0, + .transfer_len = 0, + .kind = .{ .dma_read = .{ .len = 1, .minimum_len = 1 } }, + }; + const plan = try arena.create(Plan); + plan.* = .{ + .source_slot = undefined, + .jobs = jobs, + .transfers = &.{}, + .requests = requests, + .events = events, + .source_bytes = job_count, + }; + return plan; +} + +/// Publishes a plan of `job_count` unit jobs into an open batch. +fn publishTestPlan(scheduler: *Scheduler, batch: *Batch, job_count: usize) !void { + try scheduler.publish(std.testing.io, batch, try testPlan(batch.arena.allocator(), job_count)); +} + +/// A one-file submission's sequence: publish, then drop the sentinel. +fn publishTestBatch(scheduler: *Scheduler, job_count: usize) !*Batch { + const batch = try Batch.create(std.testing.allocator, std.testing.io, 0); + errdefer batch.destroy(); + try publishTestPlan(scheduler, batch, job_count); + batch.releaseUnits(1); + return batch; +} + +/// The jobs of the queued plans no worker claimed yet. +fn unclaimedJobs(scheduler: *const Scheduler) usize { + var jobs: usize = 0; + for (0..scheduler.queue.len) |index| { + const plan = scheduler.queue.at(index); + jobs += plan.jobs.len - plan.cursor; + } + return jobs; +} + +test "scheduler publishes a submission one file at a time and claims the files in order" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + // Submitted out of file order: the sort groups "a" before "b". + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "b", .name = "b0", .shape = .init(.{4}, .u8), .offset = 0 }, + .{ .file_uri = "a", .name = "a1", .shape = .init(.{4}, .u8), .offset = 4 }, + .{ .file_uri = "a", .name = "a0", .shape = .init(.{4}, .u8), .offset = 0 }, + }; + var slots = [_]SourceSlot{ + .{ .uri = "a" }, + .{ .uri = "b" }, + }; + var outputs: [sources.len]Buffer = undefined; + var destinations: [sources.len]Destination = undefined; + var parts: [sources.len]Part = undefined; + initPlainParts(platform, &sources, &.{ &slots[1], &slots[0], &slots[0] }, &outputs, &destinations, &parts); + + var scheduler: Scheduler = .init(allocator); + defer scheduler.deinit(); + const batch = try Batch.create(allocator, io, 0); + try Planner.publishFiles(&scheduler, io, batch, &parts, .{ + .block_size = 4, + .request_size = 4, + }, fixedReads(0)); + try std.testing.expectEqual(@as(usize, 2), batch.plans.items.len); + try std.testing.expectEqual(@as(usize, 2), batch.plans.items[0].jobs.len); + try std.testing.expectEqual(@as(usize, 1), batch.plans.items[1].jobs.len); + try std.testing.expectEqual(@as(usize, 3), unclaimedJobs(&scheduler)); + try std.testing.expect(batch.published_at != null); + + // File a's jobs in offset order, then file b's. + var claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan.source_slot == &slots[0]); + try std.testing.expectEqual(@as(u64, 0), claim.plan.jobs[claim.index].file_offset); + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan.source_slot == &slots[0]); + try std.testing.expectEqual(@as(u64, 4), claim.plan.jobs[claim.index].file_offset); + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan.source_slot == &slots[1]); + try std.testing.expectEqual(@as(u64, 0), claim.plan.jobs[claim.index].file_offset); + // Both plans left the queue with their last claim; the submission is + // still open until its sentinel goes. + try std.testing.expect(scheduler.claim(io) == null); + try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); + batch.releaseUnits(1); + try std.testing.expect(!batch.done.isSet()); + batch.releaseUnits(3); + try std.testing.expect(batch.done.isSet()); + batch.destroy(); +} + +test "fifo scheduler claims plan by plan in publish order, and a done batch leaves while a later plan is queued" { + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + // An open submission's first plan, a sealed batch, then the open + // submission's second plan: an open submission holds no place of its + // own. + const open = try Batch.create(std.testing.allocator, io, 0); + try publishTestPlan(&scheduler, open, 1); + const sealed = try publishTestBatch(&scheduler, 2); + try publishTestPlan(&scheduler, open, 1); + try std.testing.expectEqual(@as(usize, 4), unclaimedJobs(&scheduler)); + + var claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan == open.plans.items[0] and claim.index == 0); + for (0..2) |index| { + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan == sealed.plans.items[0] and claim.index == index); + } + // The sealed batch left the queue with its last claim, so it completes + // and goes away while the open one's second plan is still queued. + sealed.releaseUnits(2); + try std.testing.expect(sealed.done.isSet()); + sealed.destroy(); + try std.testing.expectEqual(@as(usize, 1), scheduler.queue.len); + + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan == open.plans.items[1] and claim.index == 0); + try std.testing.expect(scheduler.claim(io) == null); + try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); + // Claimed out, the open submission completes only once its sentinel + // goes. + open.releaseUnits(2); + try std.testing.expect(!open.done.isSet()); + open.releaseUnits(1); + try std.testing.expect(open.done.isSet()); + open.destroy(); +} + +test "fifo scheduler failure retires the unclaimed jobs of every queued plan and refuses later publishes" { + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + // A batch without jobs is never queued and completes at publish. + const empty = try publishTestBatch(&scheduler, 0); + try std.testing.expect(empty.done.isSet()); + empty.destroy(); + // An open submission of two plans, then a sealed batch. + const open = try Batch.create(std.testing.allocator, io, 0); + try publishTestPlan(&scheduler, open, 2); + try publishTestPlan(&scheduler, open, 3); + const sealed = try publishTestBatch(&scheduler, 3); + try std.testing.expectEqual(@as(usize, 8), unclaimedJobs(&scheduler)); + try std.testing.expectEqual(@as(usize, 6), open.remaining.load(.acquire)); + + // The claim keeps its unit and the open submission its sentinel; every + // other job is retired. + try std.testing.expect(scheduler.claim(io).?.plan.batch == open); + scheduler.fail(io); + try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); + try std.testing.expectEqual(@as(usize, 2), open.remaining.load(.acquire)); + try std.testing.expect(sealed.done.isSet()); + sealed.destroy(); + try std.testing.expect(scheduler.claim(io) == null); + try std.testing.expect(!scheduler.waitForWork(io)); + // The submission goes on: its next plan is refused, and the sentinel + // drop leaves only the claim to complete it. + try std.testing.expectError(error.Canceled, publishTestPlan(&scheduler, open, 1)); + open.releaseUnits(1); + try std.testing.expect(!open.done.isSet()); + open.releaseUnits(1); + try std.testing.expect(open.done.isSet()); + open.destroy(); +} + +test "fifo scheduler wakes waiting workers on publish and releases them on failure" { + const WaitResult = enum(u8) { waiting, work, stopped }; + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + + var woke: std.atomic.Value(WaitResult) = .init(.waiting); + var group: std.Io.Group = .init; + const Waiter = struct { + fn run(scheduler_: *Scheduler, io_: std.Io, woke_: *std.atomic.Value(WaitResult)) void { + woke_.store(if (scheduler_.waitForWork(io_)) .work else .stopped, .release); + } + }; + try group.concurrent(io, Waiter.run, .{ &scheduler, io, &woke }); + try io.sleep(.fromMilliseconds(5), .awake); + try std.testing.expectEqual(WaitResult.waiting, woke.load(.acquire)); + const batch = try publishTestBatch(&scheduler, 1); + try group.await(io); + try std.testing.expectEqual(WaitResult.work, woke.load(.acquire)); + _ = scheduler.claim(io).?; + batch.releaseUnits(1); + batch.destroy(); + + woke.store(.waiting, .release); + try group.concurrent(io, Waiter.run, .{ &scheduler, io, &woke }); + try io.sleep(.fromMilliseconds(5), .awake); + try std.testing.expectEqual(WaitResult.waiting, woke.load(.acquire)); + scheduler.fail(io); + try group.await(io); + try std.testing.expectEqual(WaitResult.stopped, woke.load(.acquire)); +} + +test "fifo scheduler concurrent claims across two batches return each job once" { + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + const batches = [_]*Batch{ + try publishTestBatch(&scheduler, 16), + try publishTestBatch(&scheduler, 16), + }; + var seen: std.atomic.Value(u64) = .init(0); + var claim_count: std.atomic.Value(usize) = .init(0); + var duplicate: std.atomic.Value(bool) = .init(false); + var group: std.Io.Group = .init; + for (0..8) |_| try group.concurrent(io, struct { + fn run( + scheduler_: *Scheduler, + batches_: []const *Batch, + seen_: *std.atomic.Value(u64), + claim_count_: *std.atomic.Value(usize), + duplicate_: *std.atomic.Value(bool), + ) void { + while (scheduler_.claim(std.testing.io)) |claim| { + const base: u64 = if (claim.plan.batch == batches_[0]) 0 else 16; + const mask = @as(u64, 1) << @intCast(base + claim.plan.jobs[claim.index].file_offset); + if (seen_.fetchOr(mask, .acq_rel) & mask != 0) duplicate_.store(true, .release); + _ = claim_count_.fetchAdd(1, .monotonic); + } + } + }.run, .{ &scheduler, &batches, &seen, &claim_count, &duplicate }); + try group.await(io); + try std.testing.expectEqual(std.math.maxInt(u32), @as(u32, @truncate(seen.load(.acquire)))); + try std.testing.expectEqual(@as(usize, 32), claim_count.load(.acquire)); + try std.testing.expect(!duplicate.load(.acquire)); + try std.testing.expectEqual(@as(usize, 0), unclaimedJobs(&scheduler)); + for (batches) |batch| { + batch.releaseUnits(16); + try std.testing.expect(batch.done.isSet()); + batch.destroy(); + } +} + +// DMA pump and transfer target tests. + +test "device pump admits by bytes and by pieces" { + var pump: Dma.DevicePump = .{}; + try std.testing.expect(pump.hasRoom(8)); + // The piece that crosses the budget was admitted; the next is not. + pump.active_bytes = 8; + try std.testing.expect(!pump.hasRoom(8)); + pump.active_bytes = 7; + try std.testing.expect(pump.hasRoom(8)); + pump.active_pieces = max_dma_pieces_per_device; + try std.testing.expect(!pump.hasRoom(8)); +} + +test "the submission that completes a target's bytes carries the last flag" { + var target: DestinationState.Target = .{ .manager = undefined, .device_index = 0, .total = 100 }; + try std.testing.expect(!target.fullySubmitted()); + // Pieces arrive in any order; only the byte total matters. + try std.testing.expect(!target.nextIsLast(20)); + target.noteSubmitted(20); + try std.testing.expect(!target.nextIsLast(30)); + target.noteSubmitted(30); + try std.testing.expect(!target.nextIsLast(10)); + try std.testing.expect(target.nextIsLast(50)); + try std.testing.expect(!target.fullySubmitted()); + target.noteSubmitted(50); + try std.testing.expect(target.fullySubmitted()); +} + +// Request, credit and batch lifecycle tests. + +/// A single-device loader without PJRT, workers or pinned arenas: enough +/// for request, credit and batch lifecycle tests, with 64-byte blocks cut +/// as `create` cuts them, the zero block then `slots` credits of one block. +/// `platform` and `workspace` are its only undefined fields: what these +/// tests run reaches the platform only through a non-null PJRT event +/// (`Dma.EventContext.destroyEvent`). Must not move after `init`. +const TestLoader = struct { + memory: []u8, + pumps: [1]Dma.DevicePump = .{.{}}, + loader: Loader, + + fn init(self: *TestLoader, slots: usize) !void { + const allocator = std.testing.allocator; + const memory = try allocator.alloc(u8, (slots + 1) * 64); + errdefer allocator.free(memory); + const blocks = try allocator.alloc(host_memory.Block, slots + 1); + errdefer allocator.free(blocks); + for (blocks, 0..) |*block, index| block.* = memory[index * 64 ..][0..64]; + @memset(blocks[0], 0); + self.* = .{ .memory = memory, .loader = undefined }; + self.loader = .{ + .allocator = allocator, + .io = std.testing.io, + .platform = undefined, + .workspace = undefined, + .blocks = blocks, + .scheduler = .init(allocator), + .read_permits = .{ .permits = 1 }, + .read_width = 1, + .credits = try .init(allocator, blocks[1..], 1), + .pumps = &self.pumps, + .dma_budget_bytes = 64, + .created_at = .now(std.testing.io, .awake), + .plan_config = .{ .block_size = 64, .request_size = 64 }, + .reads = fixedReads(0), + }; + } + + fn deinit(self: *TestLoader) void { + const allocator = std.testing.allocator; + self.loader.scheduler.deinit(); + self.pumps[0].queue.deinit(allocator); + self.loader.credits.deinit(allocator); + allocator.free(self.loader.blocks); + allocator.free(self.memory); + } +}; + +test "a callback failing while a piece is queued drains it and completes the batch" { + const io = std.testing.io; + const allocator = std.testing.allocator; + var fixture: TestLoader = undefined; + try fixture.init(1); + defer fixture.deinit(); + const loader = &fixture.loader; + + // One request whose only piece is queued behind a DMA slot that a failed + // callback is about to free. + const batch = try publishTestBatch(&loader.scheduler, 1); + const request = loader.claimRequest().?; + var target: DestinationState.Target = .{ .manager = undefined, .device_index = 0, .total = 64 }; + _ = request.pending.fetchAdd(1, .acq_rel); + try fixture.pumps[0].queue.pushBack(allocator, .{ + .target = &target, + .request = request, + .data = loader.credits.slotBlocks(request.slot)[0], + .destination_offset = 0, + }); + fixture.pumps[0].active_bytes = 64; + fixture.pumps[0].active_pieces = 1; + request.release(); + try std.testing.expect(!batch.done.isSet()); + try std.testing.expectEqual(@as(usize, 1), loader.credits.inUse(io)); + _ = loader.first_error.set(error.Unknown); + + Dma.eventCompleted(loader, 0, 64); + try std.testing.expectEqual(@as(usize, 0), fixture.pumps[0].active_pieces); + try std.testing.expectEqual(@as(usize, 0), request.pending.load(.acquire)); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expect(batch.done.isSet()); + + Dma.retireBatch(loader, batch); + batch.destroy(); +} + +test "batch completes when every claimed request completes" { + const io = std.testing.io; + var fixture: TestLoader = undefined; + try fixture.init(3); + defer fixture.deinit(); + const loader = &fixture.loader; + + const batch = try publishTestBatch(&loader.scheduler, 3); + try std.testing.expectEqual(@as(usize, 3), batch.remaining.load(.acquire)); + var requests: [3]*ReadRequest = undefined; + for (&requests) |*request| request.* = loader.claimRequest().?; + try std.testing.expect(loader.scheduler.claim(io) == null); + try std.testing.expectEqual(@as(usize, 3), loader.credits.inUse(io)); + + for (requests, 0..) |request, index| { + try std.testing.expect(!batch.done.isSet()); + try std.testing.expectEqual(requests.len - index, batch.remaining.load(.acquire)); + request.release(); + } + try std.testing.expect(batch.done.isSet()); + try std.testing.expectEqual(@as(usize, 0), batch.remaining.load(.acquire)); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + // Each claim took its job's slot of the plan's request array. + for (requests, batch.plans.items[0].requests) |request, *slot| { + try std.testing.expect(request == slot); + } + + Dma.retireBatch(loader, batch); + batch.destroy(); +} + +test "a worker waiting for a credit when the loader fails gets one back and stops" { + const io = std.testing.io; + var fixture: TestLoader = undefined; + try fixture.init(1); + defer fixture.deinit(); + const loader = &fixture.loader; + + // A request in flight holds the only credit; a worker that saw the + // second job waits on the credits. + const batch = try publishTestBatch(&loader.scheduler, 2); + const request = loader.claimRequest().?; + try loader.worker_group.concurrent(io, Loader.workerMain, .{loader}); + while (loader.credits.condition.state.load(.monotonic).waiters == 0) io.sleep(.fromMilliseconds(1), .awake) catch {}; + + // The failure retires the second job and closes nothing: the request + // gives its credit back as it ends, the worker claims nothing with it, + // and `destroy`'s shutdown stops it. + loader.recordError(error.InputOutput, "injected source failure", .{}); + try std.testing.expect(!batch.done.isSet()); + request.release(); + loader.stopWorkers(); + try std.testing.expect(batch.done.isSet()); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expectEqual(@as(usize, 0), batch.plans.items[0].requests[1].pending.load(.acquire)); + Dma.retireBatch(loader, batch); + batch.destroy(); +} + +test "retired events are destroyed by the next pump or by a batch's retirement" { + // Without PJRT, contexts with a null event stand in for destroyed ones; + // the list mechanics are the subject. + const io = std.testing.io; + var fixture: TestLoader = undefined; + try fixture.init(2); + defer fixture.deinit(); + const loader = &fixture.loader; + + const batch = try publishTestBatch(&loader.scheduler, 2); + const requests = [_]*ReadRequest{ + loader.claimRequest().?, + loader.claimRequest().?, + }; + const plan = batch.plans.items[0]; + plan.events_used.store(2, .release); + for (plan.events, requests) |*ctx, request| ctx.* = .{ + .request = request, + .pjrt_event = null, + .device_index = 0, + .len = 64, + }; + // Two callbacks fired: the next pump destroys both, newest first. + Dma.retireEvent(loader, &plan.events[0]); + Dma.retireEvent(loader, &plan.events[1]); + const device_pump = &fixture.pumps[0]; + try std.testing.expect(device_pump.retired == &plan.events[1]); + try std.testing.expect(plan.events[1].next_retired == &plan.events[0]); + device_pump.mutex.lockUncancelable(io); + device_pump.destroyRetired(); + device_pump.mutex.unlock(io); + try std.testing.expect(device_pump.retired == null); + try std.testing.expect(plan.events[0].next_retired == null); + try std.testing.expect(plan.events[1].next_retired == null); + + // A callback that fires after the last pump leaves its context linked; + // the batch's retirement destroys it before the batch is freed, with + // the retired contexts of a batch still in flight. + var other: Dma.EventContext = .{ .request = undefined, .pjrt_event = null, .device_index = 0, .len = 64 }; + Dma.retireEvent(loader, &other); + Dma.retireEvent(loader, &plan.events[0]); + for (requests) |request| request.release(); + try std.testing.expect(batch.done.isSet()); + Dma.retireBatch(loader, batch); + try std.testing.expect(device_pump.retired == null); + try std.testing.expect(plan.events[0].next_retired == null); + try std.testing.expect(other.next_retired == null); + batch.destroy(); +} + +test "overlapping batches complete under concurrent claims and retirement" { + const io = std.testing.io; + const worker_count = 4; + var fixture: TestLoader = undefined; + try fixture.init(worker_count); + defer fixture.deinit(); + const loader = &fixture.loader; + + const Worker = struct { + fn run(loader_: *Loader) void { + while (loader_.scheduler.waitForWork(loader_.io)) if (loader_.claimRequest()) |request| request.release(); + } + }; + var group: std.Io.Group = .init; + defer { + loader.scheduler.fail(io); + group.await(io) catch {}; + } + for (0..worker_count) |_| try group.concurrent(io, Worker.run, .{loader}); + + // Three batches in flight at a time, awaited newest first: a batch is + // retired while later ones are still being claimed. + for (0..70) |round| { + var batches: [3]*Batch = undefined; + var job_counts: [3]usize = undefined; + for (&batches, &job_counts, 0..) |*batch, *job_count, offset| { + job_count.* = (round * 3 + offset) % 7; + batch.* = try publishTestBatch(&loader.scheduler, job_count.*); + } + var index = batches.len; + while (index > 0) { + index -= 1; + const batch = batches[index]; + batch.done.waitUncancelable(io); + Dma.retireBatch(loader, batch); + batch.destroy(); + } + try std.testing.expectEqual(@as(usize, 0), unclaimedJobs(&loader.scheduler)); + } + // Every request released its credit before its batch completed; the + // workers only hold one while a claim is in progress. + loader.scheduler.fail(io); + try group.await(io); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); +} + +// Source read tests. + +test "readFilePositionalAllV resumes short reads and stops at minimum" { + const OneByteIo = struct { + const file_end = 11; + + /// Reads one byte per call, the low byte of its offset, from a file + /// that ends at `file_end`, and counts the calls in `userdata`. + fn fileReadPositional(userdata: ?*anyopaque, _: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const calls: *usize = @ptrCast(@alignCast(userdata.?)); + calls.* += 1; + if (offset >= file_end) return 0; + for (data) |buffer| { + if (buffer.len == 0) continue; + buffer[0] = @truncate(offset); + return 1; + } + return 0; + } + }; + var vtable = std.testing.io.vtable.*; + vtable.fileReadPositional = OneByteIo.fileReadPositional; + var calls: usize = 0; + const io: std.Io = .{ .userdata = &calls, .vtable = &vtable }; + const file: std.Io.File = .{ .handle = 0, .flags = .{ .nonblocking = false } }; + + var output: [5]u8 = @splat(0xaa); + var buffers = [_][]u8{ output[0..1], output[1..1], output[1..3], output[3..5] }; + try std.testing.expectEqual(@as(u64, 4), try readFilePositionalAllV(io, file, &buffers, 7, 4)); + try std.testing.expectEqual(@as(usize, 4), calls); + try std.testing.expectEqualSlices(u8, &.{ 7, 8, 9, 10, 0xaa }, &output); + + var past_end = [_][]u8{&output}; + try std.testing.expectError(error.UnexpectedEndOfFile, readFilePositionalAllV(io, file, &past_end, 7, output.len)); +} + +// LazyOnce tests. + +test "lazy initialization preserves its finite error across repeated callers" { + const Context = struct { + fn init(calls: *usize) error{FileNotFound}!u32 { + calls.* += 1; + return error.FileNotFound; + } + }; + var once: LazyOnce(u32) = .{}; + var calls: usize = 0; + try std.testing.expectError(error.FileNotFound, once.ensure(std.testing.io, Context.init, .{&calls})); + try std.testing.expectError(error.FileNotFound, once.ensure(std.testing.io, Context.init, .{&calls})); + try std.testing.expectEqual(1, calls); +} + +// Whole-loader tests. + +/// Submits `specs`, which must fail with `expected`; a batch that went out +/// anyway is awaited before the test fails. +fn expectSubmitError(loader: *Loader, specs: []const LoadSpec, expected: anyerror) !void { + const batch = loader.submit(specs, .none) catch |err| return std.testing.expectEqual(expected, err); + loader.awaitBatch(batch) catch {}; + return error.TestUnexpectedResult; +} + +test "loader releases the pinned set when its workers cannot start" { + const allocator = std.testing.allocator; + const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, std.testing.io); + // The testing io, but no worker starts: `create` fails after the carve, + // the credits and the pumps, and unwinds every one. + var vtable = std.testing.io.vtable.*; + vtable.groupConcurrent = std.Io.failingGroupConcurrent; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + const result: anyerror!void = if (Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off })) |loader| unexpected: { + loader.destroy(); + break :unexpected {}; + } else |err| err; + try std.testing.expectError(error.ConcurrencyUnavailable, result); +} + +test "loader failures clean up before publication, after publication and during reading" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents = [_]u8{ 1, 2, 3, 4 }; + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "weights.bin", &contents, &path_buffer); + var later_buffer: [1024]u8 = undefined; + const later_path = try writeTestFile(tmp.dir, io, "weights.bin.later", &contents, &later_buffer); + const sources = [_]safetensors.Tensor{ + .{ .file_uri = path, .name = "first", .shape = .init(.{4}, .u8), .offset = 0 }, + .{ .file_uri = later_path, .name = "later", .shape = .init(.{5}, .u8), .offset = 0 }, + }; + const parts = [_]LoadPart{ .whole(&sources[0]), .whole(&sources[1]) }; + + // Device memory goes through blocks and DMA; host memory is written in + // place by the workers, into views of it for `host_mapped`. All clean + // up the same way. + for ([_]Memory{ .default, .host_pinned, .host_mapped }) |memory| { + // Planning allocates from the scheduler's allocator, whose + // allocations fail in turn: the first file's planning fails before + // any plan is published, then the later file's after the first + // file's plan was. Past the last, planning succeeds and the read + // fails: the later source ends after four of five bytes. + var fail_index: usize = 0; + while (true) : (fail_index += 1) { + var outputs: [2]Buffer = @splat(.{ + ._platform = platform, + ._shape = sources[0].shape, + ._shards = .empty, + }); + defer for (&outputs) |*output| output.deinit(); + var failing: std.testing.FailingAllocator = .init(allocator, .{ .fail_index = fail_index }); + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + loader.scheduler.allocator = failing.allocator(); + const specs = [_]LoadSpec{ + .{ .shape = sources[0].shape, .memory = memory, .output = &outputs[0], .parts = parts[0..1] }, + .{ .shape = sources[1].shape, .memory = memory, .output = &outputs[1], .parts = parts[1..2] }, + }; + + if (loader.submit(&specs, .none)) |batch| { + try std.testing.expectError(error.UnexpectedEndOfFile, loader.awaitBatch(batch)); + } else |err| try std.testing.expectEqual(error.OutOfMemory, err); + try std.testing.expectEqual(@as(usize, 0), loader.scheduler.queue.len); + // A worker the batch woke may still hold the credit it took for + // a claim that finds nothing: every credit is free once the + // workers stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expectEqual(loader.read_width, loader.read_permits.permits); + + // Each failure fails the loader: a later submission gets it. + try std.testing.expectEqual(@as(usize, 1), loader.batch_count); + const expected: anyerror = if (failing.has_induced_failure) error.OutOfMemory else error.UnexpectedEndOfFile; + try std.testing.expectError(expected, loader.checkOpen()); + try expectSubmitError(loader, specs[0..1], expected); + if (!failing.has_induced_failure) break; + } + } +} + +test "exactly the width of reads are in flight, and a read failing mid-load leaves nothing held" { + const allocator = std.testing.allocator; + const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, std.testing.io); + // Every source read holds for a moment and is counted; the one numbered + // `fail_at` fails. + const Reads = struct { + var in_flight: std.atomic.Value(usize) = .init(0); + var peak: std.atomic.Value(usize) = .init(0); + var started: std.atomic.Value(usize) = .init(0); + var fail_at: usize = std.math.maxInt(usize); + + fn readPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + _ = peak.fetchMax(in_flight.fetchAdd(1, .acq_rel) + 1, .acq_rel); + defer _ = in_flight.fetchSub(1, .acq_rel); + std.testing.io.sleep(.fromMilliseconds(5), .awake) catch {}; + if (started.fetchAdd(1, .acq_rel) == fail_at) return error.InputOutput; + return std.testing.io.vtable.fileReadPositional(userdata, file, data, offset); + } + }; + var vtable = std.testing.io.vtable.*; + vtable.fileReadPositional = Reads.readPositional; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + + // Sixteen tensors with gaps between them: a read, and a job, each. + const tensors = 16; + var contents: [tensors * 8]u8 = undefined; + for (&contents, 0..) |*byte, index| byte.* = @truncate(index); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, std.testing.io, "weights.bin", &contents, &path_buffer); + var sources: [tensors]safetensors.Tensor = undefined; + var parts: [tensors]LoadPart = undefined; + var specs: [tensors]LoadSpec = undefined; + var outputs: [tensors]Buffer = undefined; + for (&sources, &parts, &specs, &outputs, 0..) |*source, *part, *spec, *output, index| { + source.* = .{ .file_uri = path, .name = "t", .shape = .init(.{4}, .u8), .offset = index * 8 }; + part.* = .whole(source); + spec.* = .{ .shape = source.shape, .output = output, .parts = part[0..1] }; + } + + for ([_]?usize{ null, 5 }) |fail_at| { + for (&outputs) |*output| output.* = .{ ._platform = platform, ._shape = sources[0].shape, ._shards = .empty }; + defer for (&outputs) |*output| output.deinit(); + // `destroy` returns: no worker waits for a credit or a permit. + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + try std.testing.expectEqual(@as(usize, 2), loader.read_width); + Reads.in_flight = .init(0); + Reads.peak = .init(0); + Reads.started = .init(0); + Reads.fail_at = fail_at orelse std.math.maxInt(usize); + const batch = try loader.submit(&specs, .none); + if (fail_at == null) { + try loader.awaitBatch(batch); + // Three workers, two permits. + try std.testing.expectEqual(@as(usize, tensors), Reads.started.load(.acquire)); + try std.testing.expectEqual(loader.read_width, Reads.peak.load(.acquire)); + try testing.expectShards(allocator, std.testing.io, &outputs[tensors - 1], contents[(tensors - 1) * 8 ..][0..4], .default); + } else { + try std.testing.expectError(error.InputOutput, loader.awaitBatch(batch)); + // The reads stopped at the failure. + try std.testing.expect(Reads.started.load(.acquire) < tensors); + } + try std.testing.expectEqual(@as(usize, 0), loader.scheduler.queue.len); + // A worker the batch woke may still hold the credit it took for a + // claim that finds nothing: every credit is free once the workers + // stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expectEqual(loader.read_width, loader.read_permits.permits); + } +} + +test "a default-memory output a failed load left unfinished reports an error on every device" { + const allocator = std.testing.allocator; + // On the accelerator when it has two devices: each plugin's transfer + // manager errors the buffers it leaves unfinished. + var devices: testing.Devices = undefined; + try devices.init(allocator, std.testing.io, 2); + defer devices.deinit(); + const platform = devices.platform; + // The read reaching the last request fails. + const Reads = struct { + var fail_from: u64 = std.math.maxInt(u64); + + fn readPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + if (offset >= fail_from) return error.InputOutput; + return std.testing.io.vtable.fileReadPositional(userdata, file, data, offset); + } + }; + var vtable = std.testing.io.vtable.*; + vtable.fileReadPositional = Reads.readPositional; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + + // A replicated tensor of three requests, a job each: no device's last + // piece goes out. + const request_size = loader.plan_config.request_size; + const contents = try allocator.alloc(u8, 3 * request_size); + defer allocator.free(contents); + testing.pattern(contents, 1); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, std.testing.io, "weights.bin", contents, &path_buffer); + var source: safetensors.Tensor = .{ .file_uri = path, .name = "t", .shape = .init(.{contents.len}, .u8), .offset = 0 }; + var part: LoadPart = .whole(&source); + var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; + defer output.deinit(); + const spec: LoadSpec = .{ .shape = source.shape, .output = &output, .parts = (&part)[0..1] }; + + Reads.fail_from = 2 * request_size; + defer Reads.fail_from = std.math.maxInt(u64); + const batch = try loader.submit((&spec)[0..1], .none); + try std.testing.expectError(error.InputOutput, loader.awaitBatch(batch)); + // The failing job created the buffers before its read. Each one is + // ready, with an error, once the batch is retired: a success would + // hand out a buffer missing its last request, a pending one would hang + // its first user. + const placed = platform.placedShape(source.shape); + try std.testing.expectEqual(placed._sharding.mesh.?.devicesInCanonicalOrder().len, output._shards.len); + for (output._shards.constSlice()) |shard| { + const event = shard.readyEvent(platform.pjrt_api); + defer event.deinit(platform.pjrt_api); + try std.testing.expect(event.isReady(platform.pjrt_api)); + const err = event.getEventError(platform.pjrt_api) orelse return error.TestUnexpectedResult; + err.deinit(platform.pjrt_api); + } +} + +test "whole tensors load on four devices, sharded by rows or columns, replicated, or joined from two files" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 4); + defer devices.deinit(); + const mesh = try devices.mesh("tp"); + + // Rows of 32 bytes, 16 per device; rows cut into four pieces of 8 + // bytes, one per device; a table on every device; and a tensor whose + // rows continue in a second file. + var rows: [64 * 32]u8 = undefined; + var cols: [1024 * 32]u8 = undefined; + var table: [3000 * 4]u8 = undefined; + var tail: [64 * 32]u8 = undefined; + testing.pattern(&rows, 1); + testing.pattern(&cols, 2); + testing.pattern(&table, 3); + testing.pattern(&tail, 4); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "first.safetensors", ®istry, &.{ + .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &rows }, + .{ .name = "cols", .dtype = "U16", .dims = &.{ 1024, 16 }, .bytes = &cols }, + .{ .name = "table", .dtype = "U8", .dims = &.{ 3000, 4 }, .bytes = &table }, + }); + try testing.writeSafetensors(allocator, io, tmp.dir, "second.safetensors", ®istry, &.{ + .{ .name = "tail", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &tail }, + }); + var joined: [rows.len + tail.len]u8 = undefined; + @memcpy(joined[0..rows.len], &rows); + @memcpy(joined[rows.len..], &tail); + const parts = [_]LoadPart{ + .whole(registry.tensors.getPtr("rows").?), + .whole(registry.tensors.getPtr("cols").?), + .whole(registry.tensors.getPtr("table").?), + // rows again, then tail after it: the first file's run overlaps. + .whole(registry.tensors.getPtr("rows").?), + .{ .source = registry.tensors.getPtr("tail").?, .lattice = .{ .dst_offset = rows.len, .width = tail.len } }, + }; + var outputs: [4]Buffer = undefined; + const specs = [_]LoadSpec{ + .{ .shape = Shape.init(.{ .row = 64, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), .output = &outputs[0], .parts = parts[0..1] }, + .{ .shape = Shape.init(.{ .row = 1024, .col = 16 }, .u16).withPartitioning(&mesh, .{ .col = .model }), .output = &outputs[1], .parts = parts[1..2] }, + .{ .shape = Shape.init(.{ .row = 3000, .col = 4 }, .u8).withPartitioning(&mesh, .{}), .output = &outputs[2], .parts = parts[2..3] }, + .{ .shape = Shape.init(.{ .row = 128, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), .output = &outputs[3], .parts = parts[3..5] }, + }; + const globals = [_][]const u8{ &rows, &cols, &table, &joined }; + + // Blocks of 4 KiB and requests of two: the first file's run is cut into + // jobs, and the jobs' transfers at block ends. + const loader = try Loader.create(allocator, io, devices.platform, .{ + .read_parallelism = 2, + .read_hints = .{ .read_chunk_size = 8192 }, + .dma_block_size = 4096, + .direct_io = .off, + }); + defer loader.destroy(); + const batch = try loader.submit(&specs, .none); + var jobs: usize = 0; + for (batch.plans.items) |plan| jobs += plan.jobs.len; + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + for (&outputs, globals) |*output, global| try testing.expectShards(allocator, io, output, global, .default); + // 46,816 bytes in the first file, 2,048 in the second. + try std.testing.expectEqual(@as(usize, 6 + 1), jobs); + // Every credit is free once the workers stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); +} + +test "a submit the loader fails between its open check and its first publish returns that failure" { + const io = std.testing.io; + const platform = Platform.init(std.testing.allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(std.testing.allocator, io); + // Fails the loader in the submitting task's next allocation: after + // `submit`'s open check and before its first publish, where a worker of + // an older batch can fail it and the scheduler then answers `Canceled`. + const FailNextAllocation = struct { + parent: std.mem.Allocator, + loader: *Loader = undefined, + thread: std.Thread.Id = undefined, + armed: std.atomic.Value(bool) = .init(false), + + fn arm(self: *@This(), loader: *Loader) void { + self.loader = loader; + self.thread = std.Thread.getCurrentId(); + self.armed.store(true, .release); + } + + fn allocator(self: *@This()) std.mem.Allocator { + return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + const self: *@This() = @ptrCast(@alignCast(ctx)); + // The workers allocate on threads of their own. + if (self.armed.load(.acquire) and self.thread == std.Thread.getCurrentId()) { + self.armed.store(false, .monotonic); + self.loader.recordError(error.UnexpectedEndOfFile, "an older batch's read", .{}); + } + return self.parent.rawAlloc(len, alignment, ret_addr); + } + + fn resize(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { + const self: *@This() = @ptrCast(@alignCast(ctx)); + return self.parent.rawResize(memory, alignment, new_len, ret_addr); + } + + fn remap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + const self: *@This() = @ptrCast(@alignCast(ctx)); + return self.parent.rawRemap(memory, alignment, new_len, ret_addr); + } + + fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { + const self: *@This() = @ptrCast(@alignCast(ctx)); + self.parent.rawFree(memory, alignment, ret_addr); + } + }; + var failing: FailNextAllocation = .{ .parent = std.testing.allocator }; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "weights.bin", &.{ 1, 2, 3, 4 }, &path_buffer); + var source: safetensors.Tensor = .{ .file_uri = path, .name = "x", .shape = .init(.{4}, .u8), .offset = 0 }; + var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; + defer output.deinit(); + const loader = try Loader.create(failing.allocator(), io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + const parts = [_]LoadPart{.whole(&source)}; + failing.arm(loader); + try expectSubmitError(loader, &.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, error.UnexpectedEndOfFile); +} + +test "source slots keep their own copy of each path" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents = [_]u8{ 1, 2, 3, 4 }; + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "weights.bin", &contents, &path_buffer); + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + // Each submission names the file through a string of its own, freed + // right after the submit: workers open the file through its slot. + for (0..2) |_| { + const uri = try allocator.dupe(u8, path); + var source: safetensors.Tensor = .{ .file_uri = uri, .name = "x", .shape = .init(.{4}, .u8), .offset = 0 }; + var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; + defer output.deinit(); + const parts = [_]LoadPart{.whole(&source)}; + const submitted = loader.submit(&.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, .none); + allocator.free(uri); + try loader.awaitBatch(try submitted); + try testing.expectShards(allocator, io, &output, &contents, .default); + } + // The testing allocator overwrites freed bytes: a slot keyed by the + // first string would miss the second lookup. + try std.testing.expectEqual(@as(usize, 1), loader.source_slots.count()); +} + +test "column-sharded tensors load as one DMA piece per device and block, read exact or widened" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 4); + defer devices.deinit(); + const mesh = try devices.mesh("tp"); + + // Rows of 32 bytes, 8 per device; then 12,000 one-byte pieces, more + // than one vectored read takes. + var rows: [64 * 32]u8 = undefined; + var cols: [1024 * 32]u8 = undefined; + var narrow: [3000 * 4]u8 = undefined; + testing.pattern(&rows, 1); + testing.pattern(&cols, 2); + testing.pattern(&narrow, 3); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "staged.safetensors", ®istry, &.{ + .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &rows }, + .{ .name = "cols", .dtype = "U16", .dims = &.{ 1024, 16 }, .bytes = &cols }, + .{ .name = "narrow", .dtype = "U8", .dims = &.{ 3000, 4 }, .bytes = &narrow }, + }); + const names = [_][]const u8{ "rows", "cols", "narrow" }; + const shapes = [_]Shape{ + Shape.init(.{ .row = 64, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), + Shape.init(.{ .row = 1024, .col = 16 }, .u16).withPartitioning(&mesh, .{ .col = .model }), + Shape.init(.{ .row = 3000, .col = 4 }, .u8).withPartitioning(&mesh, .{ .col = .model }), + }; + const globals = [_][]const u8{ &rows, &cols, &narrow }; + const row_pieces = 1024 * 4 + 3000 * 4; + + for ([_]usize{ 0, 64 }) |alignment| { + var parts: [names.len]LoadPart = undefined; + var specs: [names.len]LoadSpec = undefined; + var outputs: [names.len]Buffer = undefined; + for (&parts, &specs, &outputs, names, shapes) |*part, *spec, *output, name, shape| { + part.* = .whole(registry.tensors.getPtr(name).?); + spec.* = .{ .shape = shape, .output = output, .parts = part[0..1] }; + } + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + loader.reads.alignment = alignment; + const batch = try loader.submit(&specs, .none); + // Every plan is published once `submit` returns. + var staged_jobs: usize = 0; + var submissions: usize = 0; + for (batch.plans.items) |plan| { + for (plan.jobs) |job| staged_jobs += @intFromBool(job.kind == .dma_land); + submissions += plan.events.len; + } + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + for (&outputs, globals) |*output, global| try testing.expectShards(allocator, io, output, global, .default); + try std.testing.expect(staged_jobs >= 1); + try std.testing.expect(submissions < row_pieces / 100); + // Every credit is free once the workers stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + } +} + +test "a strided part of a remote file is checked against its exact source range" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + var bytes: [128 * 128]u8 = undefined; + testing.pattern(&bytes, 1); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "square.safetensors", ®istry, &.{ + .{ .name = "square", .dtype = "U8", .dims = &.{ 128, 128 }, .bytes = &bytes }, + }); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + // The same file through a registered scheme: a remote file, read exact. + try vfs.registerBackend("remote", .{ .io = io }); + const local = registry.tensors.get("square").?; + var remote = local; + var uri_buffer: [1100]u8 = undefined; + remote.file_uri = try std.fmt.bufPrint(&uri_buffer, "remote://{s}", .{local.file_uri}); + std.debug.assert(local.offset % 4096 != 0); + + var transposed: [128 * 128]u8 = undefined; + for (0..128) |column| { + for (0..128) |row| transposed[column * 128 + row] = bytes[row * 128 + column]; + } + // Transposed: its source read whole by every job, a request exactly, + // at an offset off the direct I/O alignment. + const dims = [_]backend.Lattice.Dim{ + .{ .count = 128, .src_stride = 1, .dst_stride = 128 }, + .{ .count = 128, .src_stride = 128, .dst_stride = 1 }, + }; + const loader = try Loader.create(allocator, vfs.io(), devices.platform, .{ .read_parallelism = 2 }); + defer loader.destroy(); + try std.testing.expectEqual(VFS.direct_io_alignment, loader.reads.alignment); + loader.plan_config.request_size = bytes.len; + + var output: Buffer = undefined; + const part: LoadPart = .{ .source = &remote, .lattice = .{ .width = 1, .dims = &dims } }; + const fit = loader.stridedRead(part); + try std.testing.expectEqual(fit.limit, fit.read); + const spec: LoadSpec = .{ .shape = .init(.{ 128, 128 }, .u8), .output = &output, .parts = (&part)[0..1] }; + try loader.awaitBatch(try loader.submit(&.{spec}, .none)); + defer output.deinit(); + try testing.expectShards(allocator, io, &output, &transposed, .default); + + // A local file may be read directly: its range widened to the alignment + // no longer fits. + if (VFS.direct_io_alignment != 0) { + const local_part: LoadPart = .{ .source = &local, .lattice = .{ .width = 1, .dims = &dims } }; + const widened = loader.stridedRead(local_part); + try std.testing.expect(widened.read > widened.limit); + } +} + +/// A host-resident load on four devices: rows sharded; columns sharded, two +/// bytes per device per row, so one job has 8,192 transfers and scatters in +/// several reads; replicated; one source twice, replicated and row-sharded, +/// which overlaps; and a device-memory tensor in the same file. +const HostResidentTest = struct { + /// Each run writes its own file from `seed`: a buffer that lands on + /// memory an earlier run freed must not find the expected bytes there. + fn run(devices: *const testing.Devices, seed: u8, alignment: usize) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = devices.platform; + const mesh = try devices.mesh("tp"); + + var rows: [64 * 16]u8 = undefined; + var dev: [16 * 16]u8 = undefined; + var cols: [2048 * 8]u8 = undefined; + var rep: [96 * 4]u8 = undefined; + var twice: [32 * 4]u8 = undefined; + testing.pattern(&rows, seed); + testing.pattern(&dev, seed +% 1); + testing.pattern(&cols, seed +% 2); + testing.pattern(&rep, seed +% 3); + testing.pattern(&twice, seed +% 4); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + // The device tensor separates the host runs: an overlap sends its + // whole job through blocks, so the twice-bound source must not share + // a job with the others. + try testing.writeSafetensors(allocator, io, tmp.dir, "host.safetensors", ®istry, &.{ + .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 8 }, .bytes = &rows }, + .{ .name = "cols", .dtype = "U8", .dims = &.{ 2048, 8 }, .bytes = &cols }, + .{ .name = "rep", .dtype = "U8", .dims = &.{ 96, 4 }, .bytes = &rep }, + .{ .name = "dev", .dtype = "U16", .dims = &.{ 16, 8 }, .bytes = &dev }, + .{ .name = "twice", .dtype = "U8", .dims = &.{ 32, 4 }, .bytes = &twice }, + }); + + const names = [_][]const u8{ "rows", "dev", "cols", "rep", "twice", "twice" }; + const shapes = [_]Shape{ + Shape.init(.{ .row = 64, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }), + Shape.init(.{ .row = 16, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }), + Shape.init(.{ .row = 2048, .col = 8 }, .u8).withPartitioning(&mesh, .{ .col = .model }), + Shape.init(.{ .row = 96, .col = 4 }, .u8).withPartitioning(&mesh, .{}), + Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{}), + Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{ .row = .model }), + }; + // `rows` as views of pinned memory in the devices' default space. + const memories = [_]Memory{ .host_mapped, .default, .host_pinned, .host_pinned, .host_pinned, .host_pinned }; + const globals = [_][]const u8{ &rows, &dev, &cols, &rep, &twice, &twice }; + var parts: [names.len]LoadPart = undefined; + var specs: [names.len]LoadSpec = undefined; + var outputs: [names.len]Buffer = undefined; + for (&parts, &specs, &outputs, names, shapes, memories) |*part, *spec, *output, name, shape, memory| { + part.* = .whole(registry.tensors.getPtr(name).?); + spec.* = .{ .shape = shape, .memory = memory, .output = output, .parts = part[0..1] }; + } + + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + loader.reads.alignment = alignment; + const live_before = SharedHostAllocation.live.load(.acquire); + { + const batch = try loader.submit(&specs, .none); + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + // Host shards in each device's host memory, or its default + // memory for views. + for (&outputs, globals, memories) |*output, global, memory| try testing.expectShards(allocator, io, output, global, memory); + // Every credit, with its blocks, came back once the workers + // stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + + // The replicated host ones are one allocation. + for (outputs[3..5]) |output| { + const shards = output._shards.constSlice(); + const first = try shards[0].opaqueDeviceMemoryDataPointer(platform.pjrt_api); + for (shards[1..]) |shard| { + const pointer = try shard.opaqueDeviceMemoryDataPointer(platform.pjrt_api); + try std.testing.expect(pointer == first); + } + } + // The two replicated outputs, and one per device of `rows`. + try std.testing.expectEqual(live_before + 2 + outputs[0]._shards.len, SharedHostAllocation.live.load(.acquire)); + // Shards may be freed in any order: free one replicated output's + // last first; its allocation goes with its last view. + std.mem.reverse(*pjrt.Buffer, outputs[3]._shards.slice()); + } + // PJRT may run the views' deletion callbacks on its own threads, + // just after the shards are freed. + var polls: usize = 0; + while (SharedHostAllocation.live.load(.acquire) != live_before) : (polls += 1) { + if (polls == 5000) return error.TestUnexpectedResult; + try io.sleep(.fromMilliseconds(1), .awake); + } + } +}; + +test "host-resident destinations are written in place, through blocks when reads are widened or overlap, and replicas share one allocation" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 4); + defer devices.deinit(); + if (!backend.supportsMemory(devices.platform, .host_pinned)) return error.SkipZigTest; + // In place, then through blocks for every host job. + for ([_]usize{ 0, 64 }, [_]u8{ 1, 5 }) |alignment, seed| { + try HostResidentTest.run(&devices, seed, alignment); + } +} diff --git a/zml/io/dma_calibration.zig b/zml/io/dma_calibration.zig new file mode 100644 index 00000000..e744a9e6 --- /dev/null +++ b/zml/io/dma_calibration.zig @@ -0,0 +1,675 @@ +//! DMA sizing for the direct backend: the block size, decided once when the +//! backend is created. It screens `default_block_sizes` (2 to 32 MiB), or +//! the one block `zml.io.Loader.Options.dma_block_size` gives, on one +//! representative device, at `depth` transfers per device and over fixed +//! windows, and keeps the smallest block within 8% of the measured peak; +//! borderline candidates get longer, alternating confirmation windows. The +//! ring it measures in stays mapped as the load's first pinned capacity. +//! CPU returns the smallest candidate without measuring: its plugin's +//! transfer is a memcpy, and a load takes the same time at every block +//! size. The buffered backend has nothing to size. +//! +//! `measure` runs the same warm-up and screen without a loader, with every +//! knob, for load_check's `dma-bench` (examples/io/load_check.zig). +const std = @import("std"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const Platform = @import("../platform.zig").Platform; +const backend = @import("backend.zig"); +const host_memory = @import("host_memory.zig"); +const log = @import("log.zig").io; + +/// Blocks in flight per device: the screen runs `depth` transfers (the +/// default `Options.block_parallelism`), and the direct backend's DMA +/// budget is `depth` blocks per device, in bytes (`DevicePump.hasRoom` in +/// direct_loader.zig). Eight repeatedly won or tied on MI300X; wider stages +/// mostly increased callback latency and pinned memory, and an adaptive DMA +/// width added state with little load benefit. At 32, four GB300 loaded +/// DeepSeek 9% faster for 2.5x the pinned memory. +pub const depth: usize = 8; + +/// Screened, not fixed: eight MI300X loaded replicated Gemma in 7.83 s at +/// 16 MiB against 10.69 s at 8, whereas one B70 lost 10.5% at 16 MiB. +pub const default_block_sizes = [_]usize{ + 2 * 1024 * 1024, + 4 * 1024 * 1024, + 8 * 1024 * 1024, + 16 * 1024 * 1024, + 32 * 1024 * 1024, +}; + +/// The screen's knobs. The loader sets only `block_sizes` and keeps the +/// other defaults; `measure` takes them all. +pub const Options = struct { + block_sizes: []const usize = &default_block_sizes, + /// Transfers in flight per device in the screen, at most + /// `max_block_parallelism`. The loader keeps `depth`. + block_parallelism: usize = depth, + /// A screen window runs for at least this long and, unless the target is + /// zero, until the representative device completes the transfer target. + /// On eight MI300X, 2 ms/32-transfer windows calibrate in 0.96 s against + /// 4.83 s at 10 ms/128, selecting the same block. Short screens sometimes + /// selected the wrong block under noise, hence the longer borderline + /// confirmation windows (`confirmation_duration`). + minimum_duration: std.Io.Duration = .fromMilliseconds(2), + minimum_transfers: u64 = 32, + /// Prefer the smallest block within this tolerance of the measured peak + /// transfer throughput. + block_selection_tolerance: f64 = 0.08, +}; + +/// Borderline block candidates receive longer paired windows, alternating +/// which runs first. A candidate is borderline when its rounds disagree on +/// whether it is near the peak, or when its median ratio to the peak lies +/// within `confirmation_margin` of the tolerance's floor. +const confirmation_duration: std.Io.Duration = .fromMilliseconds(25); +const confirmation_minimum_transfers: u64 = 256; +const confirmation_margin: f64 = 0.02; + +/// The widest screen `measure` takes. +pub const max_block_parallelism: usize = 32; + +/// Measures as the direct loader's creation does, without a loader and with +/// every knob of `opts`, and returns the block it selects: warms the device +/// allocators, then screens the blocks in a workspace of its own, placed +/// near the devices as for reads through the page cache (it reads no file), +/// released before it returns. CPU returns the smallest block without +/// measuring; a platform that loads through the buffered backend +/// (`backend.loadsDirect`) returns `UnsupportedPlatform`, and a knob out of +/// range `InvalidOptions`. Every failure is logged. For load_check's +/// `dma-bench`. +pub fn measure(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options) !usize { + try validate(opts); + if (!backend.loadsDirect(platform)) { + log.err("DMA calibration: {t} loads through the buffered backend, which has no DMA to measure: UnsupportedPlatform", .{platform.target}); + return error.UnsupportedPlatform; + } + platform.warmupDeviceAllocators(io) catch |err| { + log.failure(err, "DMA calibration: warm the device allocators: target={t}", .{platform.target}); + return err; + }; + var workspace = try host_memory.Workspace.init(allocator, io, platform, false); + defer workspace.deinit(); + return calibrate(&workspace, platform, opts) catch |err| { + log.failure(err, "DMA calibration: target={t}", .{platform.target}); + return err; + }; +} + +/// A block is at most the largest source request, the bound +/// `zml.io.Loader.init` puts on its own block sizes, so every +/// candidate's ring of `block_parallelism` blocks fits the workspace's +/// mapped ceiling. +fn validate(opts: Options) error{InvalidOptions}!void { + comptime std.debug.assert(backend.max_read_request_size * max_block_parallelism <= host_memory.Workspace.mapped_bytes_ceiling); + if (opts.block_parallelism == 0 or opts.block_parallelism > max_block_parallelism) { + log.err("invalid DMA calibration options: block_parallelism={d}, expected 1..{d}: InvalidOptions", .{ opts.block_parallelism, max_block_parallelism }); + return error.InvalidOptions; + } + if (opts.block_sizes.len == 0) { + log.err("invalid DMA calibration options: block_sizes is empty: InvalidOptions", .{}); + return error.InvalidOptions; + } + for (opts.block_sizes) |block_size| { + if (block_size == 0 or block_size > backend.max_read_request_size) { + log.err("invalid DMA calibration options: block size {d}, expected 1..{d}: InvalidOptions", .{ block_size, backend.max_read_request_size }); + return error.InvalidOptions; + } + } + if (opts.minimum_duration.nanoseconds <= 0) { + log.err("invalid DMA calibration options: minimum_duration={f}, expected more than zero: InvalidOptions", .{opts.minimum_duration}); + return error.InvalidOptions; + } + // Written so that NaN fails too. + if (!(opts.block_selection_tolerance >= 0 and opts.block_selection_tolerance < 1)) { + log.err("invalid DMA calibration options: block_selection_tolerance={d}, expected [0, 1): InvalidOptions", .{opts.block_selection_tolerance}); + return error.InvalidOptions; + } +} + +/// Measures one representative device into the direct loader's workspace +/// and returns the block it selects; CPU returns the smallest without +/// measuring. On a platform mixing device kinds, device 0's block serves +/// every device: it is a tuning choice, valid for all of them. `opts` were +/// checked (`validate`, or `zml.io.Loader.init` for the loader's block +/// sizes). The rate the `dma_bench` line logs is that one device's +/// synthetic H2D rate, not the platform's nor a prediction of a +/// checkpoint's load speed. +pub fn calibrate( + workspace: *host_memory.Workspace, + platform: *const Platform, + opts: Options, +) !usize { + // Every candidate loads alike on CPU: four PJRT CPU devices loaded warm + // sharded Llama in 1.55-1.60 s at 2, 8 and 16 MiB, so the smallest, + // which maps the least, is taken unmeasured and the loader grows its + // own arenas. + if (platform.target == .cpu) return std.mem.min(usize, opts.block_sizes); + + const allocator = workspace.allocator; + const io = workspace.io; + const start: std.Io.Timestamp = .now(io, .awake); + + var max_total_bytes: usize = 0; + for (opts.block_sizes) |block_size| { + max_total_bytes = @max(max_total_bytes, block_size * opts.block_parallelism); + } + // Map one ring for the largest candidate at the fixed transfer width; + // every candidate reuses it, and the load keeps it. + const calibration_source = try workspace.allocate(max_total_bytes); + + var session = try Session.init(allocator, io, platform, opts.block_sizes, opts.block_parallelism); + defer session.deinit(); + + const candidates = try allocator.alloc(Candidate, opts.block_sizes.len); + defer allocator.free(candidates); + for (candidates, opts.block_sizes, 0..) |*candidate, block_size, index| { + candidate.* = .{ .block_size = block_size, .buffer_offset = index * opts.block_parallelism }; + } + try measureAllCandidates(&session, opts, candidates, calibration_source); + const winner = candidates[try selectCandidate(&session, opts, candidates, calibration_source)]; + + log.debug("dma_bench platform={s} devices={d} kind=\"{s}\" block_bytes={d} parallelism={d} measured_gib_s={d:.3} duration={f} retained_mapped_bytes={d}", .{ + @tagName(platform.target), + platform.devices.len, + platform.devices[0].kind(), + winner.block_size, + opts.block_parallelism, + winner.median().bytesPerSecond() / (1024 * 1024 * 1024), + start.durationTo(.now(io, .awake)), + workspace.mappedBytes(), + }); + return winner.block_size; +} + +const sample_count = 3; + +fn measureAllCandidates( + session: *Session, + opts: Options, + candidates: []Candidate, + source: []const u8, +) !void { + for (0..sample_count) |round| { + for (0..candidates.len) |i| { + const candidate = &candidates[(i + round) % candidates.len]; + candidate.metrics[round] = try runWindow( + session.io, + session, + candidate, + source, + opts.minimum_duration, + opts.minimum_transfers, + ); + } + } +} + +/// Returns the index of the selected candidate (`pick`), over each +/// candidate's median ratio to the peak, or its confirmation's when it is +/// borderline. +fn selectCandidate( + session: *Session, + opts: Options, + candidates: []Candidate, + source: []const u8, +) !usize { + const ratios = try session.allocator.alloc(f64, candidates.len); + defer session.allocator.free(ratios); + + var peak_index: usize = 0; + var peak_rate = candidates[0].median().bytesPerSecond(); + for (candidates[1..], 1..) |candidate, index| { + const rate = candidate.median().bytesPerSecond(); + if (rate > peak_rate) { + peak_index = index; + peak_rate = rate; + } + } + for (candidates, ratios) |candidate, *ratio| { + ratio.* = if (peak_rate == 0) 0 else candidate.median().bytesPerSecond() / peak_rate; + } + + // A borderline candidate's ratio is its confirmation's against the peak. + for (candidates, ratios, 0..) |*candidate, *ratio, index| { + if (index != peak_index and isBorderline(candidates, index, ratio.*, opts.block_selection_tolerance)) + ratio.* = try confirmCandidateAgainstPeak(session, candidate, &candidates[peak_index], source); + } + return pick(candidates, ratios, opts.block_selection_tolerance); +} + +/// The smallest block whose ratio is within `tolerance` of the best ratio, +/// screened or confirmed. A confirmation can rate a block above the screen's +/// peak, so the best ratio may belong to a larger block. +fn pick(candidates: []const Candidate, ratios: []const f64, tolerance: f64) usize { + var best_index: usize = 0; + for (ratios, 0..) |ratio, index| { + if (ratio > ratios[best_index]) best_index = index; + } + const floor = ratios[best_index] * (1.0 - tolerance); + var selected_index = best_index; + for (candidates, ratios, 0..) |candidate, ratio, index| { + if (ratio >= floor and candidate.block_size < candidates[selected_index].block_size) + selected_index = index; + } + return selected_index; +} + +/// Whether `candidates[index]`, at screen `ratio` to the peak, gets +/// confirmation windows (`confirmation_margin`). +fn isBorderline(candidates: []const Candidate, index: usize, ratio: f64, tolerance: f64) bool { + // Whether the candidate is near-peak in one round but fails in others. + var qualified_once = false; + var rejected_once = false; + for (candidates[index].metrics, 0..) |metric, round| { + var round_peak_rate: f64 = 0; + for (candidates) |round_candidate| { + round_peak_rate = @max(round_peak_rate, round_candidate.metrics[round].bytesPerSecond()); + } + const round_ratio = if (round_peak_rate == 0) 0 else metric.bytesPerSecond() / round_peak_rate; + if (round_ratio >= 1.0 - tolerance) + qualified_once = true + else + rejected_once = true; + } + return (qualified_once and rejected_once) or @abs(ratio - (1.0 - tolerance)) <= confirmation_margin; +} + +/// The median, over `sample_count` paired windows, of `candidate`'s rate +/// over `peak`'s. +fn confirmCandidateAgainstPeak( + session: *Session, + candidate: *Candidate, + peak: *Candidate, + source: []const u8, +) !f64 { + var ratios: [sample_count]f64 = undefined; + for (&ratios, 0..) |*ratio, repeat| { + const order = if (repeat % 2 == 0) + [_]*Candidate{ candidate, peak } + else + [_]*Candidate{ peak, candidate }; + var candidate_rate: f64 = undefined; + var peak_rate: f64 = undefined; + for (order) |measured| { + const rate = (try runWindow( + session.io, + session, + measured, + source, + confirmation_duration, + confirmation_minimum_transfers, + )).bytesPerSecond(); + if (measured == candidate) candidate_rate = rate else peak_rate = rate; + } + ratio.* = if (peak_rate == 0) 0 else candidate_rate / peak_rate; + } + std.mem.sort(f64, &ratios, {}, std.sort.asc(f64)); + return ratios[ratios.len / 2]; +} + +/// Runs one window of `candidate`'s transfers, `session.parallelism` at a +/// time. `io` sleeps and joins the window; the workers transfer on the +/// session's. +fn runWindow( + io: std.Io, + session: *Session, + candidate: *Candidate, + source: []const u8, + minimum_duration: std.Io.Duration, + minimum_transfers: u64, +) !Measurement { + const Worker = struct { + session: *Session, + buffer_index: usize, + block: []const u8, + metrics: *Counters, + ready: *std.atomic.Value(usize), + start: *std.Io.Event, + stop: *std.atomic.Value(bool), + + fn run(self: @This()) void { + _ = self.ready.fetchAdd(1, .release); + self.start.waitUncancelable(self.session.io); + while (!self.stop.load(.acquire)) { + self.session.transfer(self.buffer_index, self.block, self.metrics); + if (self.session.first_error.get() != null) return; + } + } + }; + + var metrics: Counters = .{}; + var ready: std.atomic.Value(usize) = .init(0); + var start: std.Io.Event = .unset; + var stop: std.atomic.Value(bool) = .init(false); + var group: std.Io.Group = .init; + defer { + // Workers borrow this window's stack. Every exit, including a canceled + // sleep or partial spawn, must release the start gate and join them. + stop.store(true, .release); + start.set(io); + group.await(io) catch {}; + } + for (0..session.parallelism) |slot| { + try group.concurrent(io, Worker.run, .{Worker{ + .session = session, + .buffer_index = candidate.buffer_offset + slot, + .block = candidate.block(source, slot), + .metrics = &metrics, + .ready = &ready, + .start = &start, + .stop = &stop, + }}); + } + + try session.warmup(candidate, source); + + while (ready.load(.acquire) != session.parallelism) try io.sleep(.fromMicroseconds(100), .awake); + const start_at: std.Io.Timestamp = .now(io, .awake); + start.set(io); + while (true) { + const elapsed = start_at.durationTo(.now(io, .awake)); + if (session.first_error.get()) |err| return err; + // is window complete + if (elapsed.nanoseconds >= minimum_duration.nanoseconds and metrics.transfers.load(.acquire) >= minimum_transfers) break; + try io.sleep(.fromMilliseconds(1), .awake); + } + stop.store(true, .release); + try group.await(io); + const elapsed = start_at.durationTo(.now(io, .awake)); + if (session.first_error.get()) |err| return err; + return .{ .bytes = metrics.bytes.load(.acquire), .elapsed = elapsed }; +} + +/// One block size and its screen measurements. Its slots are the session's +/// buffers from `buffer_offset` on, one per transfer in flight. +const Candidate = struct { + block_size: usize, + buffer_offset: usize, + /// Slots warmed so far (`Session.warmup`). + warmed_buffers: usize = 0, + + /// One window per screen round (`measureAllCandidates`). + metrics: [sample_count]Measurement = undefined, + + fn median(self: Candidate) Measurement { + var sorted = self.metrics; + std.mem.sort(Measurement, &sorted, {}, struct { + fn lessThan(_: void, lhs: Measurement, rhs: Measurement) bool { + return lhs.bytesPerSecond() < rhs.bytesPerSecond(); + } + }.lessThan); + return sorted[sorted.len / 2]; + } + + /// The ring's bytes that `slot` transfers. + fn block(self: Candidate, source: []const u8, slot: usize) []const u8 { + return source[slot * self.block_size ..][0..self.block_size]; + } +}; + +/// One transfer manager for the whole screen, sliced per candidate, and the +/// screen's first transfer error, which ends it. +const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + manager: *pjrt.AsyncHostToDeviceTransferManager, + buffers: []*pjrt.Buffer, + /// Each candidate's slots, and the transfers a window runs at a time. + parallelism: usize, + first_error: stdx.FirstError(pjrt.ApiError) = .{}, + + fn transfer(self: *Session, buffer_index: usize, block: []const u8, metrics: ?*Counters) void { + const api = self.platform.pjrt_api; + const event = self.manager.transferData(api, buffer_index, block, 0, false) catch |err| { + _ = self.first_error.set(err); + return; + }; + defer event.deinit(api); + event.await(api, self.io) catch |err| { + _ = self.first_error.set(err); + return; + }; + if (metrics) |output| { + _ = output.bytes.fetchAdd(@intCast(block.len), .monotonic); + _ = output.transfers.fetchAdd(1, .monotonic); + } + } + + /// Transfers twice into each of `candidate`'s slots not yet warmed. + fn warmup(self: *Session, candidate: *Candidate, source: []const u8) !void { + while (candidate.warmed_buffers < self.parallelism) : (candidate.warmed_buffers += 1) { + const slot = candidate.warmed_buffers; + self.transfer(candidate.buffer_offset + slot, candidate.block(source, slot), null); + self.transfer(candidate.buffer_offset + slot, candidate.block(source, slot), null); + if (self.first_error.get()) |err| return err; + } + } + + fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + block_sizes: []const usize, + parallelism: usize, + ) !Session { + const buffer_count = std.math.mul(usize, block_sizes.len, parallelism) catch return error.OutOfMemory; + const dims = try allocator.alloc([1]i64, block_sizes.len); + defer allocator.free(dims); + const specs = try allocator.alloc(pjrt.ShapeSpec, buffer_count); + defer allocator.free(specs); + for (block_sizes, dims, 0..) |block_size, *dim, index| { + dim.* = .{@intCast(block_size)}; + @memset(specs[index * parallelism ..][0..parallelism], pjrt.ShapeSpec.init(dim, .u8)); + } + const buffers = try allocator.alloc(*pjrt.Buffer, buffer_count); + errdefer allocator.free(buffers); + const manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ + .shape_specs = specs, + .memory = platform.devices[0].memory(.default).?.pjrt_memory, + }); + var retrieved: usize = 0; + errdefer { + manager.deinit(platform.pjrt_api); + for (buffers[0..retrieved]) |buffer| buffer.deinit(platform.pjrt_api); + } + for (buffers, 0..) |*buffer, index| { + buffer.* = try manager.retrieveBuffer(platform.pjrt_api, index); + retrieved += 1; + } + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .manager = manager, + .buffers = buffers, + .parallelism = parallelism, + }; + } + + fn deinit(self: *Session) void { + // XLA commit 1b19ae012aa67426658f7ca3c1503bb781c863a9 switched GPU + // transfers to CommonAsyncHostToDeviceTransferManager, whose destructor + // waits for outstanding transfers and marks unfinished buffers as errored. + // We discard these buffers, so no final is_last_transfer=true copy is needed. + self.manager.deinit(self.platform.pjrt_api); + for (self.buffers) |buffer| buffer.deinit(self.platform.pjrt_api); + self.allocator.free(self.buffers); + self.* = undefined; + } +}; + +const Counters = struct { + bytes: std.atomic.Value(u64) = .init(0), + transfers: std.atomic.Value(u64) = .init(0), +}; + +const Measurement = struct { + bytes: u64, + elapsed: std.Io.Duration, + + fn bytesPerSecond(self: Measurement) f64 { + if (self.elapsed.nanoseconds <= 0) return 0; + return @as(f64, @floatFromInt(self.bytes)) * std.time.ns_per_s / + @as(f64, @floatFromInt(self.elapsed.nanoseconds)); + } +}; + +test "DMA benchmark selection uses medians and prefers the smallest near-peak value" { + // No candidate below is borderline, so the selector never schedules a + // confirmation window and the session supplies only its allocator. + var session: Session = .{ + .allocator = std.testing.allocator, + .io = undefined, + .platform = undefined, + .manager = undefined, + .buffers = undefined, + .parallelism = undefined, + }; + const opts: Options = .{ .block_selection_tolerance = 0.05 }; + + var candidates = [_]Candidate{ + .{ .block_size = 2, .buffer_offset = undefined }, + .{ .block_size = 4, .buffer_offset = undefined }, + .{ .block_size = 8, .buffer_offset = undefined }, + }; + const rates = [_][sample_count]u64{ + .{ 60, 10, 62 }, + .{ 98, 99, 97 }, + .{ 100, 101, 99 }, + }; + for (&candidates, rates) |*candidate, candidate_rates| { + for (&candidate.metrics, candidate_rates) |*metric, rate| { + metric.* = .{ .bytes = rate, .elapsed = .fromSeconds(1) }; + } + } + const winner = candidates[try selectCandidate(&session, opts, &candidates, &.{})]; + try std.testing.expectEqual(@as(usize, 4), winner.block_size); + try std.testing.expectEqual(@as(f64, 98), winner.median().bytesPerSecond()); + + // A larger block near the peak in every round is neither confirmed nor + // picked. + var bimodal = [_]Candidate{ + .{ .block_size = 2, .buffer_offset = undefined }, + .{ .block_size = 4, .buffer_offset = undefined }, + .{ .block_size = 8, .buffer_offset = undefined }, + .{ .block_size = 16, .buffer_offset = undefined }, + }; + const bimodal_rates = [_]u64{ 80, 100, 70, 99 }; + for (&bimodal, bimodal_rates) |*candidate, rate| { + candidate.metrics = @splat(.{ .bytes = rate, .elapsed = .fromSeconds(1) }); + } + try std.testing.expectEqual(@as(usize, 4), bimodal[try selectCandidate(&session, opts, &bimodal, &.{})].block_size); +} + +test "DMA benchmark picks the smallest block within the tolerance of the best ratio" { + const candidates = [_]Candidate{ + .{ .block_size = 16 * 1024 * 1024, .buffer_offset = undefined }, + .{ .block_size = 32 * 1024 * 1024, .buffer_offset = undefined }, + }; + // 32 MiB is the peak; 16 MiB within the tolerance below it, then past it. + try std.testing.expectEqual(0, pick(&candidates, &.{ 0.93, 1.0 }, 0.08)); + try std.testing.expectEqual(1, pick(&candidates, &.{ 0.91, 1.0 }, 0.08)); + // 16 MiB is the screen's peak; a confirmation rates 32 MiB more than the + // tolerance above it, then within it. + try std.testing.expectEqual(1, pick(&candidates, &.{ 1.0, 1.10 }, 0.08)); + try std.testing.expectEqual(0, pick(&candidates, &.{ 1.0, 1.05 }, 0.08)); +} + +test "DMA benchmark confirms a candidate when round qualification disagrees" { + var candidates = [_]Candidate{ + .{ .block_size = 4, .buffer_offset = undefined }, + .{ .block_size = 8, .buffer_offset = undefined }, + }; + const rates = [_][sample_count]u64{ + .{ 99, 80, 70 }, + .{ 100, 100, 100 }, + }; + for (&candidates, rates) |*candidate, candidate_rates| { + for (&candidate.metrics, candidate_rates) |*metric, rate| { + metric.* = .{ .bytes = rate, .elapsed = .fromSeconds(1) }; + } + } + // The median ratio, 0.80, is far from the 0.95 floor: only the rounds + // make it borderline. + try std.testing.expect(isBorderline(&candidates, 0, 0.80, 0.05)); +} + +test "measure refuses a knob out of range before it touches the platform" { + const cases = [_]Options{ + .{ .block_sizes = &.{} }, + .{ .block_sizes = &.{0} }, + .{ .block_sizes = &.{backend.max_read_request_size + 1} }, + .{ .block_parallelism = 0 }, + .{ .block_parallelism = max_block_parallelism + 1 }, + .{ .minimum_duration = .zero }, + .{ .minimum_duration = .fromMilliseconds(-1) }, + .{ .block_selection_tolerance = 1 }, + .{ .block_selection_tolerance = -0.01 }, + .{ .block_selection_tolerance = std.math.nan(f64) }, + }; + for (cases) |opts| { + try std.testing.expectError(error.InvalidOptions, measure(std.testing.allocator, std.testing.io, undefined, opts)); + } +} + +test "measure needs no loader and selects the smallest block on CPU" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + try std.testing.expectEqual(4 << 20, try measure(allocator, io, platform, .{ .block_sizes = &.{ 8 << 20, 4 << 20 } })); +} + +test "DMA benchmark shares one manager across candidate sizes and slots" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.auto(allocator, io, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + var session = try Session.init(allocator, io, platform, &.{ 8, 16 }, 2); + defer session.deinit(); + const source: [32]u8 = @splat(0); + for ([_]usize{ 8, 16 }, 0..) |block_size, index| { + var candidate: Candidate = .{ .block_size = block_size, .buffer_offset = index * 2 }; + try session.warmup(&candidate, &source); + try std.testing.expectEqual(2, candidate.warmed_buffers); + } + try std.testing.expectEqual(null, session.first_error.get()); +} + +test "DMA benchmark cancellation drains transfer workers" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.auto(allocator, io, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + var source: [16]u8 = @splat(0); + var session = try Session.init(allocator, io, platform, &.{source.len}, 1); + defer session.deinit(); + var candidate: Candidate = .{ .block_size = source.len, .buffer_offset = 0 }; + var vtable = io.vtable.*; + vtable.sleep = struct { + fn sleep(_: ?*anyopaque, _: std.Io.Timeout) std.Io.Cancelable!void { + return error.Canceled; + } + }.sleep; + const canceled_io: std.Io = .{ .userdata = io.userdata, .vtable = &vtable }; + + // A cancellation can arrive while workers wait for the start gate or + // while they transfer. In either case they must finish before teardown. + try std.testing.expectError(error.Canceled, runWindow( + canceled_io, + &session, + &candidate, + &source, + .max, + 0, + )); +} diff --git a/zml/io/host_memory.zig b/zml/io/host_memory.zig new file mode 100644 index 00000000..45063b7e --- /dev/null +++ b/zml/io/host_memory.zig @@ -0,0 +1,613 @@ +//! Loader-owned host arenas, cut into blocks once. The direct backend owns +//! its workspace for its whole life: calibration maps its ring there, the +//! backend grows the set to what its load holds and carves it, and its +//! `destroy` releases every arena. Callers never supply a workspace, a +//! budget, a NUMA policy or a calibration result. +//! +//! The memory depends on the target: CUDA and oneAPI take page-backed arenas +//! registered with PJRT (DmaMap), ROCm and Furiosa PJRT `pinned_host` +//! buffers, and CPU plain pages, unregistered. The arenas' kind and the +//! transfer path are separate decisions: CPU loads through the direct +//! backend too. When the task's cpuset and memory policy allow several +//! memory-bearing NUMA nodes, page-backed arenas interleave over those of the +//! platform's devices (all of them when the load reads past the page cache, +//! a device's node is unknown, or none of theirs is allowed); ROCm balances +//! whole arenas across its devices' host nodes. +//! `Workspace.mapped_bytes_ceiling` (16 GiB) is a safety guard, not a +//! target. A failed allocation or registration, or a growth past the +//! ceiling, is logged with its cause and returns `OutOfMemory`. + +const std = @import("std"); +const Alignment = std.mem.Alignment; +const Allocator = std.mem.Allocator; +const builtin = @import("builtin"); + +const pjrt = @import("pjrt"); + +const Platform = @import("../platform.zig").Platform; +const log = @import("log.zig").mem; + +/// One ROCm host-memory allocation path and its bytes allocated so far. +const HostNode = struct { + /// The plugin's `numa_node` device attribute (`Device.numaNode`); null + /// for the one path kept when no device reports it. + node: ?u32, + any_device_index: usize, + total_allocated_bytes: usize = 0, +}; + +/// One retained arena. The bytes of a `pinned_host` arena belong to its +/// PJRT buffer; page-backed arenas have none. +const Arena = struct { + bytes: []u8, + pjrt_buffer: ?*pjrt.Buffer = null, +}; + +/// How arenas are mapped and unmapped; each strategy keeps only the state +/// it uses. +const ArenaKind = union(enum) { + /// With eight MI300X visible, hipHostRegister took ~6.3 s for 1 GiB + /// versus 0.6-0.7 s through hipHostMalloc. KFD/IOMMU registration maps + /// pages to every GPU; huge-page advice helped only ~9%, and touching + /// pages or changing HIP flags did not remove the cost. Standard PJRT + /// pinned_host buffers provide allocation-owned memory without a custom + /// allocator extension. + pjrt_host: struct { + platform: *const Platform, + host_nodes: std.ArrayList(HostNode), + }, + /// Page-backed arenas, registered with the platform's PJRT client when + /// there is one (`dma_map` in the arena log) and left pageable for CPU + /// transfers when not (`pageable`), with huge-page alignment and advice. + /// Registration only avoids staging if the plugin recognizes the range as + /// pinned, including subranges. The oneAPI plugin must implement + /// IsHostMemoryPinned for SYCL imports: without it, one B70 staged + /// transfers at 11.01 GiB/s instead of 26.90 (eight DMA caps). + pages: struct { + platform: ?*const Platform, + /// Zero leaves placement to the kernel. Automatic placement can + /// fall back to zero if the kernel refuses it. + numa_mask: u64, + }, + + const transparent_huge_page_size = 2 * 1024 * 1024; + const mpol_interleave = 3; + + fn init( + allocator: Allocator, + io: std.Io, + platform: *const Platform, + direct_reads: bool, + ) Allocator.Error!ArenaKind { + return switch (platform.target) { + .cuda, .oneapi => .{ .pages = .{ .platform = platform, .numa_mask = pagesNumaMask(allocator, io, platform, direct_reads) } }, + .cpu => .{ .pages = .{ .platform = null, .numa_mask = pagesNumaMask(allocator, io, platform, direct_reads) } }, + // Furiosa backs `.host_pinned` with SDK pinned (dma-buf) memory + // from `/dev/dma_heap/system`, which the SDK recognizes by + // address and transfers without pinning pages on every request. + .rocm, .furiosa => { + // PJRT-pinned allocations choose placement through the device's + // host memory space, not our mbind policy. Retain one allocation + // path per reported node; the blocks stay unpartitioned. + var host_nodes: std.ArrayList(HostNode) = try .initCapacity(allocator, @max(platform.devices.len, 1)); + devices: for (platform.devices, 0..) |device, device_index| { + std.debug.assert(device.memory(.host_pinned) != null); + const node = device.numaNode() orelse continue; + for (host_nodes.items) |existing| { + if (existing.node == node) continue :devices; + } + host_nodes.appendAssumeCapacity(.{ .node = node, .any_device_index = device_index }); + } + if (host_nodes.items.len == 0) host_nodes.appendAssumeCapacity(.{ .node = null, .any_device_index = 0 }); + return .{ .pjrt_host = .{ .platform = platform, .host_nodes = host_nodes } }; + }, + .tpu, .neuron, .metal => @panic("DMA is not accessible on this platform and shouldn't have been called."), + }; + } + + /// Call after every arena is unmapped. + fn deinit(self: *ArenaKind, allocator: Allocator) void { + switch (self.*) { + .pjrt_host => |*host| host.host_nodes.deinit(allocator), + .pages => {}, + } + } + + /// Maps one arena of `bytes`. + fn map(self: *ArenaKind, allocator: Allocator, io: std.Io, bytes: usize) Allocator.Error!Arena { + const started: std.Io.Timestamp = .now(io, .awake); + switch (self.*) { + .pjrt_host => |*host| { + // Balance bytes, not arena/device counts: arenas differ in size. + // Replicated Llama-3.1-8B on eight MI300X lost the fast mode when + // all arenas came from device zero (1.27-1.45 s). Device rotation + // left a 61/39 byte split and took 0.98-1.33 s; byte-balanced + // allocation recovered 0.84-1.05 s over six runs, with 1.52 GiB + // mapped versus 2.03 GiB for per-node pools. Host contention + // affected both arms. + var host_node = &host.host_nodes.items[0]; + for (host.host_nodes.items[1..]) |*candidate| { + if (candidate.total_allocated_bytes < host_node.total_allocated_bytes) host_node = candidate; + } + const device_index = host_node.any_device_index; + // PJRT is the sole owner of these hipHostMalloc-backed bytes. + // Never DmaMap or DmaUnmap the borrowed pointer: unregistering + // allocation-owned memory breaks the eventual hipHostFree. The + // buffer and its external reference both live until `unmap`, + // after all reads and DMA finish. + // The ROCm plugin must implement IsHostMemoryPinned: without + // it, one MI300X staged transfers at 6.5 GiB/s instead of 46.6. + const api = host.platform.pjrt_api; + const memory = host.platform.devices[device_index].memory(.host_pinned).?; + const buffer = host.platform.pjrt_client.createUninitializedBuffer(api, .{ + .dims = &.{@intCast(bytes)}, + .element_type = .u8, + .layout = .{ + .tiled = .{ + .minor_to_major = &.{0}, + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .dst = .{ .memory = memory.pjrt_memory }, + }) catch |err| { + log.err("pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); + return error.OutOfMemory; + }; + errdefer buffer.deinit(api); + std.debug.assert(buffer.isOnCpu(api)); + buffer.increaseExternalReferenceCount(api) catch |err| { + log.err("retaining pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); + return error.OutOfMemory; + }; + errdefer buffer.decreaseExternalReferenceCount(api) catch {}; + const ptr: [*]u8 = @ptrCast(buffer.opaqueDeviceMemoryDataPointer(api) catch |err| { + log.err("accessing pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); + return error.OutOfMemory; + }); + // Counted once mapped: the workspace retains every mapped arena. + host_node.total_allocated_bytes += bytes; + log.debug("DMA arena kind=pjrt_host device={d} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ + device_index, + @intFromPtr(ptr), + bytes, + @as(f64, @floatFromInt(elapsedNanoseconds(started, .now(io, .awake)))) / std.time.ns_per_ms, + }); + return .{ .bytes = ptr[0..bytes], .pjrt_buffer = buffer }; + }, + .pages => |*pages| { + const alignment = pageAlignment(bytes); + const ptr = allocator.rawAlloc(bytes, alignment, @returnAddress()) orelse { + log.err("host memory allocation failed for {Bi:.2}: OutOfMemory", .{bytes}); + return error.OutOfMemory; + }; + const data = ptr[0..bytes]; + place(&pages.numa_mask, data); + adviseHugePages(data); + if (pages.platform) |p| p.pjrt_client.dmaMap(p.pjrt_api, data) catch |err| { + log.err("DMA registration failed for {Bi:.2} of host memory: {s}", .{ bytes, @errorName(err) }); + allocator.rawFree(data, alignment, @returnAddress()); + return error.OutOfMemory; + }; + const finished: std.Io.Timestamp = .now(io, .awake); + log.debug("DMA arena kind={s} placement={s} nodes=0x{x} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ + if (pages.platform != null) "dma_map" else "pageable", + if (pages.numa_mask == 0) "unplaced" else "interleave", + pages.numa_mask, + @intFromPtr(data.ptr), + bytes, + @as(f64, @floatFromInt(elapsedNanoseconds(started, finished))) / std.time.ns_per_ms, + }); + return .{ .bytes = data }; + }, + } + } + + fn unmap(self: *const ArenaKind, allocator: Allocator, arena: Arena) void { + switch (self.*) { + .pjrt_host => |host| { + const api = host.platform.pjrt_api; + const buffer = arena.pjrt_buffer.?; + buffer.decreaseExternalReferenceCount(api) catch unreachable; + buffer.deinit(api); + }, + .pages => |pages| { + if (pages.platform) |platform| { + // Unregistering can fail after a device error. The pages may + // still be registered, so they are leaked, not reused. + platform.pjrt_client.dmaUnmap(platform.pjrt_api, arena.bytes) catch |err| { + log.warn("DMA unregistration failed for {Bi:.2} of host memory, which stays allocated: {s}", .{ arena.bytes.len, @errorName(err) }); + return; + }; + } + allocator.rawFree(arena.bytes, pageAlignment(arena.bytes.len), @returnAddress()); + }, + } + } + + /// Page alignment, raised to the huge-page size for arenas that span one. + fn pageAlignment(len: usize) Alignment { + const alignment: Alignment = comptime .fromByteUnits(std.heap.page_size_min); + if (comptime builtin.os.tag != .linux) return alignment; + if (len < transparent_huge_page_size) return alignment; + return alignment.max(.fromByteUnits(transparent_huge_page_size)); + } + + /// Interleaves `data` over `numa_mask`'s nodes; on refusal, warns and + /// zeroes the mask, so later arenas stay unplaced too. + fn place(numa_mask: *u64, data: []u8) void { + const mask = numa_mask.*; + if (mask == 0) return; + if (comptime builtin.os.tag != .linux) return; + + const node_mask: [1]u64 = .{mask}; + const highest_node: usize = 63 - @clz(mask); + const rc = std.os.linux.syscall6( + .mbind, + @intFromPtr(data.ptr), + data.len, + mpol_interleave, + @intFromPtr(&node_mask), + // Linux get_nodes() decrements maxnode before copying the mask; + // raw callers include the same extra sentinel bit as libnuma. + highest_node + 2, + 0, + ); + if (std.os.linux.errno(rc) == .SUCCESS) return; + log.warn("NUMA placement of DMA arenas over nodes 0x{x} refused ({s}); leaving them unplaced", .{ + mask, + @tagName(std.os.linux.errno(rc)), + }); + // Automatic placement is not retried: the arenas stay unplaced. + numa_mask.* = 0; + } + + fn adviseHugePages(data: []u8) void { + if (comptime builtin.os.tag != .linux) return; + if (data.len < transparent_huge_page_size) return; + + const ptr: [*]align(std.heap.page_size_min) u8 = @alignCast(data.ptr); + std.posix.madvise(ptr, data.len, std.posix.MADV.HUGEPAGE) catch |err| { + log.warn("MADV_HUGEPAGE failed for DMA buffer at 0x{x} ({Bi:.2}): {s}", .{ + @intFromPtr(data.ptr), + data.len, + @errorName(err), + }); + }; + } +}; + +/// Host arenas owned by one direct loader. Calibration maps its ring +/// (`allocate`); the loader then grows the set to what its load holds and +/// cuts it into blocks, once (`carve`). Deinitialize after all transfers +/// finish and before the platform. +pub const Workspace = struct { + /// Safety guard on the arenas' total host memory (pinned on the DMA + /// targets), not an allocation target: calibration and the load's + /// pre-growth decide what is actually mapped. Fixed, because no caller + /// ever had a reason to choose another value. + pub const mapped_bytes_ceiling: usize = 16 * 1024 * 1024 * 1024; + + allocator: Allocator, + io: std.Io, + kind: ArenaKind, + /// Every arena mapped, in mapping order, until `deinit`. + arenas: std.ArrayList(Arena) = .empty, + /// The arenas cut into blocks, set by `carve`: nothing maps after it. + blocks: ?[]Block = null, + + /// `direct_reads`: the load reads its checkpoint past the page cache, so + /// page-backed arenas spread over every candidate node + /// (`placementMask`). + pub fn init( + allocator: Allocator, + io: std.Io, + platform: *const Platform, + direct_reads: bool, + ) Allocator.Error!Workspace { + return .{ + .allocator = allocator, + .io = io, + .kind = try .init(allocator, io, platform, direct_reads), + }; + } + + pub fn deinit(self: *Workspace) void { + const io = self.io; + const mapped_bytes = self.mappedBytes(); + const started: std.Io.Timestamp = .now(io, .awake); + if (self.blocks) |blocks| self.allocator.free(blocks); + for (self.arenas.items) |arena| self.kind.unmap(self.allocator, arena); + self.arenas.deinit(self.allocator); + self.kind.deinit(self.allocator); + const elapsed_ns = elapsedNanoseconds(started, .now(io, .awake)); + log.debug("DMA load workspace teardown: mapped={Bi:.2}, elapsed_ms={d:.3}", .{ + mapped_bytes, + @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms, + }); + self.* = undefined; + } + + /// The bytes of every retained arena, tails included. + pub fn mappedBytes(self: *const Workspace) usize { + var mapped: usize = 0; + for (self.arenas.items) |arena| mapped += arena.bytes.len; + return mapped; + } + + /// Retains one new arena within the mapped-byte ceiling, before `carve`. + pub fn allocate(self: *Workspace, bytes: usize) Allocator.Error![]u8 { + std.debug.assert(self.blocks == null); + const mapped = self.mappedBytes(); + if (bytes > mapped_bytes_ceiling - mapped) { + log.err("host memory allocation exceeds mapped ceiling: requested={Bi:.2}, mapped={Bi:.2}, ceiling={Bi:.2}: OutOfMemory", .{ bytes, mapped, mapped_bytes_ceiling }); + return error.OutOfMemory; + } + // Reserved first, so a mapped arena is always retained: the ROCm + // balancer counts the bytes it maps. + try self.arenas.ensureUnusedCapacity(self.allocator, 1); + const arena = try self.kind.map(self.allocator, self.io, bytes); + self.arenas.appendAssumeCapacity(arena); + return arena.bytes; + } + + /// The blocks of `block_size` the workspace can hold: those of its + /// retained arenas, and those the mapped ceiling still allows. + pub fn capacity(self: *const Workspace, block_size: usize) usize { + return self.usableBlocks(block_size) + (mapped_bytes_ceiling - self.mappedBytes()) / block_size; + } + + /// Maps the blocks missing below `first_blocks` as one arena, then those + /// missing below `total_blocks` as another, and cuts every retained arena + /// into blocks of `block_size`, once. The set is then complete: nothing + /// maps inside a load (a slab mapped there cost 146 to 230 ms of + /// hipHostMalloc on MI300X), and `allocate` asserts it. Arena tails + /// smaller than one block stay mapped and unused. Blocks are not split + /// per NUMA node: strict per-node pools duplicated source reserves and + /// constrained reads to the smallest node; placement belongs to arena + /// allocation. The workspace owns the list and the memory. + pub fn carve(self: *Workspace, block_size: usize, first_blocks: usize, total_blocks: usize) Allocator.Error![]const Block { + std.debug.assert(self.blocks == null and block_size != 0); + for ([_]usize{ first_blocks, total_blocks }) |target_blocks| { + const missing_blocks = target_blocks -| self.usableBlocks(block_size); + if (missing_blocks != 0) _ = try self.allocate(missing_blocks * block_size); + } + const blocks = try self.allocator.alloc(Block, self.usableBlocks(block_size)); + var next: usize = 0; + for (self.arenas.items) |arena| { + for (0..arena.bytes.len / block_size) |j| { + blocks[next] = arena.bytes[j * block_size ..][0..block_size]; + next += 1; + } + } + std.debug.assert(next == blocks.len); + self.blocks = blocks; + return blocks; + } + + /// Counts complete blocks across retained arenas; block_size must be nonzero. + fn usableBlocks(self: *const Workspace, block_size: usize) usize { + var usable: usize = 0; + for (self.arenas.items) |arena| usable += arena.bytes.len / block_size; + return usable; + } + + /// Creates ordinary allocator-backed arenas for tests without a PJRT platform. + fn initForTesting(allocator: Allocator, io: std.Io) Workspace { + return .{ + .allocator = allocator, + .io = io, + .kind = .{ .pages = .{ .platform = null, .numa_mask = 0 } }, + }; + } +}; + +pub const Block = []u8; + +/// Bits of `/sys/devices/system/node/has_memory`, or zero when unreadable. +/// Nodes 64 and above cannot be represented and are dropped. +/// Device-coherent HBM nodes are not candidates for host pages: the measured +/// four-GB300 topology had 34 NUMA nodes but only two with host memory. +fn memoryNodeMask(allocator: Allocator, io: std.Io) u64 { + if (comptime builtin.os.tag != .linux) return 0; + const contents = std.Io.Dir.cwd().readFileAlloc( + io, + "/sys/devices/system/node/has_memory", + allocator, + .limited(4096), + ) catch return 0; + defer allocator.free(contents); + return parseNodeList(contents); +} + +/// The nodes the task's memory policy allows (`numactl --membind`, +/// `--interleave`, `--preferred`); every node under the default policy. +/// An arena placed outside them would defeat the policy: `mbind` on a +/// range overrides `set_mempolicy`. +fn taskMemoryPolicyMask() u64 { + if (comptime builtin.os.tag != .linux) return std.math.maxInt(u64); + var mode: c_int = 0; + var nodes: [1]u64 = .{0}; + const rc = std.os.linux.syscall5(.get_mempolicy, @intFromPtr(&mode), @intFromPtr(&nodes), 64, 0, 0); + if (std.os.linux.errno(rc) != .SUCCESS or mode == 0 or nodes[0] == 0) return std.math.maxInt(u64); + return nodes[0]; +} + +/// The nodes the task's cpuset lets it allocate on (`Mems_allowed`), or +/// every node when unknown. `mbind` refuses a mask with none of them. +fn cpusetMemsMask() u64 { + if (comptime builtin.os.tag != .linux) return std.math.maxInt(u64); + const mpol_f_mems_allowed = 4; + var nodes: [1]u64 = .{0}; + const rc = std.os.linux.syscall5(.get_mempolicy, 0, @intFromPtr(&nodes), 64, 0, mpol_f_mems_allowed); + if (std.os.linux.errno(rc) != .SUCCESS or nodes[0] == 0) return std.math.maxInt(u64); + return nodes[0]; +} + +/// The nodes page-backed arenas interleave over (`placementMask`): the +/// memory-bearing nodes the task's cpuset and memory policy allow, narrowed +/// to the nodes of the platform's devices, unless the load reads past the +/// page cache. +fn pagesNumaMask(allocator: Allocator, io: std.Io, platform: *const Platform, direct_reads: bool) u64 { + const candidates = memoryNodeMask(allocator, io) & cpusetMemsMask() & taskMemoryPolicyMask(); + return placementMask(candidates, deviceNodeMask(platform), direct_reads); +} + +/// The nodes of the platform's devices (`Device.numaNode`), or null when a +/// device reports none (the CPU plugin never does) or a node past 63. +fn deviceNodeMask(platform: *const Platform) ?u64 { + var mask: u64 = 0; + for (platform.devices) |device| { + const node = device.numaNode() orelse return null; + if (node >= 64) return null; + mask |= @as(u64, 1) << @intCast(node); + } + return if (mask == 0) null else mask; +} + +/// Zero leaves placement to the kernel: with one candidate node or none +/// there is nothing to choose, and the task's policy or the only node decides. +/// For reads through the page cache, the arenas go to the devices' nodes +/// among several candidates: an arena on another node crosses the +/// inter-socket link on every H2D copy. Two GB300 on one Grace node loaded +/// Qwen3.8-27B-FP8 in 1.265 s with a task policy holding the arenas to that +/// node, against 1.64 s interleaved over both nodes. Devices on several +/// nodes share one pool interleaved over theirs: four GB300 on two nodes +/// loaded replicated DeepSeek-V4-Flash in 4.35 s against 4.92 s with +/// per-node pools. When a device's node is unknown, or none is a candidate, +/// the arenas interleave over every candidate, which no single node beat +/// everywhere. A single node is placed too, since unplaced CUDA arenas +/// follow the registering thread's node; the kernel still falls back when +/// it is full. +/// A load read past the page cache (`direct_reads`) spreads the arenas over +/// every candidate instead: the drives write into them, and interleaving +/// splits that traffic, and the copies out of them, between the candidates' +/// memory. Over an md RAID0 of four NVMe drives, two on each node, the same +/// two GB300 loaded the same checkpoint forced direct in 1.752 s with the +/// arenas interleaved over both nodes, against 1.873 s with them on the +/// devices' node: staging a read took 0.52 against 0.64 ms, and a 16 MiB +/// read 1.90 against 1.87 ms. The drives' own nodes are not consulted: every +/// candidate matched them on the one host measured. +fn placementMask(candidates: u64, device_nodes: ?u64, direct_reads: bool) u64 { + if (@popCount(candidates) < 2) return 0; + if (direct_reads) return candidates; + const near = candidates & (device_nodes orelse candidates); + return if (near != 0) near else candidates; +} + +/// Parses a kernel node list ("0-1", "0,2-3") into a mask; zero on any error. +fn parseNodeList(text: []const u8) u64 { + var mask: u64 = 0; + var ranges = std.mem.tokenizeAny(u8, text, ", \n"); + while (ranges.next()) |range| { + var ends = std.mem.splitScalar(u8, range, '-'); + const first = std.fmt.parseInt(usize, ends.first(), 10) catch return 0; + const last = if (ends.next()) |end| + std.fmt.parseInt(usize, end, 10) catch return 0 + else + first; + if (last < first or ends.next() != null) return 0; + var node = first; + while (node <= last) : (node += 1) { + if (node < 64) mask |= @as(u64, 1) << @intCast(node); + } + } + return mask; +} + +fn elapsedNanoseconds(started: std.Io.Timestamp, finished: std.Io.Timestamp) u64 { + return @intCast(@max(started.durationTo(finished).nanoseconds, 0)); +} + +test "automatic NUMA placement keeps the devices' nodes for buffered reads and spreads direct reads over every candidate" { + // One candidate or none: the kernel places. + try std.testing.expectEqual(@as(u64, 0), placementMask(0, 0b10, false)); + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1, false)); + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1000, null, false)); + // Devices on one of two nodes: that node; on both: both. + try std.testing.expectEqual(@as(u64, 0b10), placementMask(0b11, 0b10, false)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11, false)); + // Unknown device nodes, or none among the candidates: every candidate. + try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null, false)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b100, false)); + // A cpuset of nodes 0 and 2 of four, devices on node 1: the cpuset's + // nodes, as `pagesNumaMask` narrows the candidates to them first. + try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10, false)); + + // Direct reads: every candidate, wherever the devices are. + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b10, true)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11, true)); + try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null, true)); + // Still nothing to choose with one candidate, and still within the + // cpuset. + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1, true)); + try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10, true)); +} + +test "parseNodeList accepts kernel node lists" { + try std.testing.expectEqual(@as(u64, 0b11), parseNodeList("0-1\n")); + try std.testing.expectEqual(@as(u64, 0b1101), parseNodeList("0,2-3")); + try std.testing.expectEqual(@as(u64, 0b1), parseNodeList("0")); + try std.testing.expectEqual(@as(u64, 0), parseNodeList("")); + try std.testing.expectEqual(@as(u64, 0), parseNodeList("1-0")); + try std.testing.expectEqual(@as(u64, 0), parseNodeList("x")); + try std.testing.expectEqual(@as(u64, 0b1), parseNodeList("0,64")); +} + +test "Workspace capacity counts whole blocks under the mapped ceiling, and carve refuses one more" { + const allocator = std.testing.allocator; + var workspace: Workspace = .initForTesting(allocator, std.testing.io); + defer workspace.deinit(); + const ceiling = Workspace.mapped_bytes_ceiling; + + try std.testing.expectEqual(ceiling / 64, workspace.capacity(64)); + _ = try workspace.allocate(150); + _ = try workspace.allocate(127); + // Whole blocks only: the arenas hold three, their tails none. + try std.testing.expectEqual(3 + (ceiling - 277) / 64, workspace.capacity(64)); + // Capacity is tight, as direct `create` sizes its carve from it: one + // block more crosses the ceiling (reaching it would map 16 GiB). + try std.testing.expectError(error.OutOfMemory, workspace.allocate(ceiling - 277 + 1)); + try std.testing.expectError(error.OutOfMemory, workspace.carve(64, workspace.capacity(64) + 1, 0)); + try std.testing.expectEqual(@as(usize, 277), workspace.mappedBytes()); +} + +test "Workspace carve maps each step's missing blocks as one arena and leaves tails out" { + const allocator = std.testing.allocator; + var workspace: Workspace = .initForTesting(allocator, std.testing.io); + defer workspace.deinit(); + const first = try workspace.allocate(150); + const second = try workspace.allocate(127); + + // Three blocks are mapped: two more make the first five, three more the + // eight. + const blocks = try workspace.carve(64, 5, 8); + try std.testing.expectEqual(@as(usize, 8), blocks.len); + try std.testing.expectEqual(@as(usize, 4), workspace.arenas.items.len); + try std.testing.expectEqual(@as(usize, 2 * 64), workspace.arenas.items[2].bytes.len); + try std.testing.expectEqual(@as(usize, 3 * 64), workspace.arenas.items[3].bytes.len); + for (blocks) |block| try std.testing.expectEqual(@as(usize, 64), block.len); + // Tails stay mapped without contributing a block. + try std.testing.expectEqual(first.ptr, blocks[0].ptr); + try std.testing.expectEqual(first.ptr + 64, blocks[1].ptr); + try std.testing.expectEqual(second.ptr, blocks[2].ptr); +} + +test "Workspace arena ownership cleans up allocation failures" { + const AllocationTest = struct { + fn run(allocator: Allocator) !void { + var workspace: Workspace = .initForTesting(allocator, std.testing.io); + defer workspace.deinit(); + + _ = try workspace.allocate(64); + _ = try workspace.allocate(128); + // The first step is already mapped, as calibration's ring covers + // one device's reserve; the second maps the last block. + const blocks = try workspace.carve(64, 3, 4); + try std.testing.expectEqual(@as(usize, 4), blocks.len); + try std.testing.expectEqual(@as(usize, 256), workspace.mappedBytes()); + try std.testing.expectEqual(@as(usize, 3), workspace.arenas.items.len); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, AllocationTest.run, .{}); +} diff --git a/zml/io/loader.zig b/zml/io/loader.zig new file mode 100644 index 00000000..7d63c4d8 --- /dev/null +++ b/zml/io/loader.zig @@ -0,0 +1,2888 @@ +//! Checkpoint loading. `Loader.start` takes a whole model: it starts +//! compiling the programs that compute some of its tensors +//! (`Source.bindProgram`) and publishes every tensor the loader fills +//! itself. `wait` then publishes the programs' inputs in groups the devices +//! have room for, runs each group's programs on the caller's task once its +//! inputs landed, and retires everything. Backends only read sources into +//! buffers; lookup, compilation, admission and execution stay in this +//! shared front end. The contract is in the pub doc comments below; +//! docs/learn/loader.md is the user guide. +//! +//! From bindings to specs. `start` checks every tensor and program input +//! once, before anything is published (`prepareSpec`, `checkPlacement`). +//! It lowers each bound source to bytes (`Source.Placed.lower`, the one +//! rule), one part per piece: a part of the destination's logical bytes +//! (the row-major bytes of its shape, before the sharding splits them), a +//! run of `width` bytes repeated over strided dimensions. A whole +//! checkpoint tensor is one part. +//! A strided part of at most `max_part_runs` runs, or of runs of at least +//! `large_run_bytes`, becomes one part per run; a spec with any other +//! strided part (row interleaves, transposes, broadcasts) goes to the +//! backend as strided parts, which the direct backend gathers. The parts go +//! in destination order, and the bytes none covers are zero. Each +//! `LoadSpec` is one destination, which the backends trust as +//! `backend.LoadSpec` states. Only `Source` makes bindings and its pieces +//! are disjoint by construction: nothing here checks that (Source's tests +//! do). +//! +//! Order and admission. The tensors the loader fills go out first, as one +//! submission, so every program group is admitted with them counted. A +//! group holds the tensors of consecutive programs in model order, up to +//! `program_group_bytes` of outputs, so the inputs of one layer's experts, +//! adjacent in the checkpoint, are read together. Before publishing a +//! group, `wait` waits for its programs to compile, then retires the oldest +//! pending submissions until the group fits the room every device reports +//! (`admission`): its limit, minus what is in use, minus the bytes the +//! direct backend published but has not allocated yet, minus a 64 MiB +//! reserve. A device's allocator may count freed memory as in use until it +//! is given back (`Platform.releaseFreedDeviceMemory`), so `start` gives +//! back what was freed before the load. A group costs its inputs, its +//! outputs and the largest of its programs' temporaries per device, since +//! they run one after the other. Once no program group is pending, retiring +//! frees nothing more: the room is read again after the retired groups' +//! memory is given back, and the group goes even if it is still short, +//! since the room is an estimate and can read low. A group that does not +//! fit a device fails its allocation or its execution with the device's +//! error, which `wait` returns; a smaller `Options.program_group_bytes` +//! makes smaller groups. Where a device reports no limit (CPU) or the +//! backend does not count its allocations (buffered), nothing is measured +//! and one program group is in flight at a time. +const std = @import("std"); + +const pjrt = @import("pjrt"); +const VFS = @import("vfs"); + +const Buffer = @import("../buffer.zig").Buffer; +const Compiler = @import("../Compiler.zig"); +const Exe = @import("../exe.zig").Exe; +const mem = @import("../mem.zig"); +const Bufferized = mem.Bufferized; +const meta = @import("../meta.zig"); +const ops = @import("../ops.zig"); +const platform_mod = @import("../platform.zig"); +const Memory = TensorStore.Memory; +const Platform = platform_mod.Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; +const testing = @import("../testing.zig"); +const backend = @import("backend.zig"); +const Backend = backend.Backend; +const buffered_loader = @import("buffered_loader.zig"); +const LoadPart = backend.LoadPart; +const LoadSpec = backend.LoadSpec; +const DeviceBytes = backend.DeviceBytes; +const direct_loader = @import("direct_loader.zig"); +const load_log = @import("log.zig").load; +const ShardGrid = @import("ShardGrid.zig"); +const TensorStore = @import("TensorStore.zig"); + +const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; + +/// Loads models from their checkpoint sources. A value: `init` returns at +/// once. +/// +/// What it borrows. The platform outlives the loader and every buffer it +/// loads. `io`, and the VFS behind it, outlive the loader: its tasks run on +/// `io`, and the direct backend opens each source file through it and +/// keeps it open until `deinit`, under a copy of the file's path, so +/// nothing of the store is kept past `wait`. `start` borrows the store, the +/// buffers and the progress node until `wait` or `cancel` returns, and +/// reads the model during the call only. The loaded buffers are the +/// caller's and outlive the loader. +/// +/// Start and wait serially on the owning task; source reads and transfers +/// run concurrently inside the backend, compiles on tasks of their own, or +/// inline when `io` cannot run one concurrently. Submissions are retired in +/// publish order. +/// +/// What fails. Every error of `start` and `wait` fails the loader: +/// everything started is canceled (`cancel`), and every later call returns +/// the first error. Each is logged once, at err, where it is detected, but +/// for a plain allocation failure (`OutOfMemory` from an allocator) and +/// `Canceled`, the caller's own `cancel`, which are returned unlogged. A +/// model whose bindings are wrong is `InvalidBinding`, logged with the +/// tensor's field path, the key its source reads (none for a source that +/// reads nothing or a run inside a byte) and the rule it breaks, or for a +/// program whose output differs, the program's name and the shape it +/// computes; buffers shaped unlike the model (`mem.pairBufferized`) too, +/// logged with the field path where they differ. The other errors a caller +/// can act on: a missing checkpoint file is `FileNotFound`; a denied one +/// `AccessDenied` locally and `PermissionDenied` remotely; a file that ends +/// inside a tensor `UnexpectedEndOfFile`, because it changed after its +/// header was read; host memory the backend cannot allocate, pin or fit +/// under its mapped ceiling `OutOfMemory`; device memory that runs out +/// PJRT's `ResourceExhausted`; an io that cannot run the loader's tasks +/// concurrently `ConcurrencyUnavailable`; `cancel` `Canceled`. The rest +/// (PJRT, the compiler, a remote that failed) passes through as its source +/// names it. +pub const Loader = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + /// `init`'s task, which warms the device allocators and creates the + /// backend, until the first `start` awaits it. + setup: ?std.Io.Future(anyerror!Backend), + /// What `setup` created; null before the first `start`, and after a + /// failed set-up. + backend: ?Backend = null, + /// `Options.program_group_bytes`. + program_group_bytes: usize, + /// The programs of the loads started since the last `wait`, each + /// compiled once. + programs: std.ArrayList(*CompiledProgram) = .empty, + /// Program groups started and not yet published, oldest first. + queued: std.Deque(Group) = .empty, + /// Submissions published and not yet retired, oldest first. + pending: std.Deque(PendingSubmission) = .empty, + /// The first error; every later call returns it. + failure: ?anyerror = null, + /// The direct backend when admission can measure the room: it counts + /// its allocations and every device reported an allocator limit at the + /// first `start`. Otherwise one program group is in flight at a time. + measured: ?*direct_loader.Loader = null, + stats: Stats = .{}, + + pub const Options = struct { + /// The checkpoint's path or URI, read by `init` only, which copies a + /// local one the VFS may read direct for the set-up task + /// (`backend.Options.checkpoint`). When `io` is a VFS, the VFS + /// backend serving it sizes the reads (`VFS.readHints`), and the + /// direct backend decides at set-up how such a local one's files + /// read, and places its pinned memory for that + /// (`direct_loader.Loader.create`). Null, or a plain `io`: local + /// sizing, pinned memory near the devices. + checkpoint: ?[]const u8 = null, + /// Concurrent source reads, 1 to `max_read_parallelism`, fixed for + /// the load. Null: 16 for a local checkpoint, 32 for a high-latency + /// one. The direct backend may lower it to fit its pinned memory + /// (logged at debug). + read_parallelism: ?usize = null, + /// The direct backend's DMA block, at most 32 MiB, and a multiple of + /// `VFS.direct_io_alignment` when `io` is a VFS and `direct_io` is + /// not `off`. Null: calibration picks one of + /// `dma_calibration.default_block_sizes`, which suits every measured + /// host. A tuning knob, ignored by the buffered backend; `init` + /// refuses a block the checkpoint's reads cannot use. + dma_block_size: ?usize = null, + /// Direct I/O for local source files, decided once per file per + /// loader (`VFS.DirectIo` for the page-cache trade-off). Nothing + /// changes when `io` is not a VFS. + direct_io: VFS.DirectIo = .auto, + /// Output bytes of the program tensors one group holds at most, one + /// tensor at least. A group's inputs are read together and freed + /// once its programs ran. + program_group_bytes: usize = 8 << 30, + + /// The default read widths by latency class, from the recorded width + /// sweeps. Local sources knee at 12 to 16 on every host measured: the + /// B70 peaks at 12 to 16 and loses 11% at 24, one MI300X prefers 12, + /// the RTX 5090 host 16 over 12, and the GB300 plateau from 12 to 48 + /// has 16 within 5% of its best rung, so 16 is the width with the + /// smallest regret. Remote sources plateau from 24 to 32: hf:// reads + /// 32 and 64 alike and AWS was flat from 24, while pinned memory grows + /// with the width (32 x 32 MiB requests pin 1.5 GiB). + const local_read_parallelism: usize = 16; + const high_latency_read_parallelism: usize = 32; + }; + + /// Concurrent source reads at most (`Options.read_parallelism`). + pub const max_read_parallelism: usize = 128; + + /// Checks `opts` and returns at once. Warming the device allocators and + /// creating the backend (for the direct one: DMA calibration and the + /// pinned pre-growth) run on a task of the loader's own, or here when + /// `io` has no concurrency; the first `start` waits for that task and + /// returns its error. A read width or a block size out of range, a + /// block that is not a multiple of the direct-I/O alignment + /// (`Options.dma_block_size`), a checkpoint whose scheme the VFS does + /// not serve, a block calibration may select whose request (the read + /// chunk rounded up to whole blocks) passes 32 MiB, or a request too + /// short for a widened direct read is `InvalidOptions`, logged. A copy + /// of the path of a local checkpoint the VFS may read direct + /// (`Options.checkpoint`) that cannot be allocated is `OutOfMemory`, + /// unlogged. + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Options, + ) error{ InvalidOptions, OutOfMemory }!Loader { + var backend_opts = try backendOptions(io, opts); + if (backend_opts.checkpoint) |path| backend_opts.checkpoint = try allocator.dupe(u8, path); + const setup_args = .{ allocator, io, platform, backend_opts }; + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .setup = io.concurrent(setUp, setup_args) catch io.async(setUp, setup_args), + .program_group_bytes = opts.program_group_bytes, + }; + } + + /// `init`'s task: the device allocators warmed, then the backend. It + /// owns `opts.checkpoint`, `init`'s copy, and runs to its end even when + /// `deinit` cancels it. + fn setUp(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: backend.Options) anyerror!Backend { + defer if (opts.checkpoint) |path| allocator.free(path); + platform.warmupDeviceAllocators(io) catch |err| { + load_log.failure(err, "warm the device allocators", .{}); + return err; + }; + return Backend.init(allocator, io, platform, opts); + } + + /// Waits for `init`'s task the first time: its backend, then whether + /// admission can measure the devices; or its error, for the caller to + /// fail the loader with. + fn awaitSetup(self: *Loader) !void { + const setup = if (self.setup) |*task| task else return; + const result = setup.await(self.io); + self.setup = null; + self.backend = try result; + self.measured = self.probeMemory(); + } + + /// Gives up on everything started: drops the queued groups, aborts the + /// backend (queued reads are dropped, reads in flight finish), retires + /// the pending submissions without running their programs, and waits + /// for the compiles. Each buffer of the started loads is then empty or + /// holds its shards, whose contents are unspecified (unwritten, errored, + /// or partly written); read none, and free the tree as after a load + /// (`mem.deinitBufferized`). The loader is failed afterwards. For an + /// error between `start` and `wait`; nothing to do when nothing was + /// started. + pub fn cancel(self: *Loader) void { + if (self.loadsStarted()) self.fail(error.Canceled); + } + + /// Cancels `init`'s task when no `start` awaited it, destroying what it + /// built, and destroys the backend: its host memory, its workers and the + /// files it keeps open, closed through `io`. The buffers it loaded stay + /// the caller's. Panics on loads neither waited for nor canceled. + pub fn deinit(self: *Loader) void { + if (self.loadsStarted()) std.debug.panic("Loader.deinit with loads started, whose store and buffers may be freed already: add `errdefer loader.cancel();` after `start`", .{}); + self.stats.logAdmission(self.measured != null); + self.queued.deinit(self.allocator); + self.pending.deinit(self.allocator); + self.programs.deinit(self.allocator); + if (self.setup) |*setup| { + if (setup.cancel(self.io)) |backend_| backend_.destroy() else |_| {} + } + if (self.backend) |backend_| backend_.destroy(); + self.* = undefined; + } + + /// Whether loads are started and neither waited for nor canceled. + fn loadsStarted(self: *const Loader) bool { + return self.queued.len != 0 or self.pending.len != 0 or self.programs.items.len != 0; + } + + /// The direct backend when every device reports an allocator limit + /// (`measured`). Decided once: the CPU plugin logs an unimplemented + /// warning on every stats call, so admission never asks a device that + /// answered nothing. + fn probeMemory(self: *const Loader) ?*direct_loader.Loader { + const direct = switch (self.backend.?) { + .direct => |direct| direct, + .buffered => return null, + }; + for (self.platform.devices) |device| { + if (device.memoryStats().bytes_limit == null) return null; + } + return direct; + } + + /// Starts loading every tensor of `model` into its buffer in `buffers`, + /// as `store` binds it, and returns; `wait` returns once it is done. + /// Each tensor is placed by its shape's sharding (replicated without a + /// mesh), into the memory its binding names, assembled from the pieces + /// of its source (`Source.Placed.bind`) with every byte no piece covers + /// set to zero. All of these are published here, as one submission. A + /// tensor a program computes (`Source.bindProgram`) starts its program + /// compiling, once for every tensor of an equal program over the same + /// placed shapes; `wait` lands its inputs and runs it. + /// + /// Every tensor and program input is checked once, before anything is + /// published: a tensor `store` did not bind, a source that reads no + /// bytes (an empty checkpoint tensor among them), a source that this + /// loader cannot assemble (a run that starts or ends inside a byte, + /// strided parts into host memory, a slice along an inner axis, a + /// broadcast or a transpose whose source range does not fit the direct + /// backend's smallest read: the checkpoint's read chunk (8 MiB locally), + /// rounded up to whole blocks of the smallest it may pick, 2 MiB or a + /// pinned `dma_block_size`), a sub-byte last dimension that does not + /// pack into whole bytes, a placement its mesh does not divide or on + /// devices the platform lacks, and a host kind the platform cannot take + /// are `error.InvalidBinding`. + /// + /// `buffers` is `mem.bufferize` of `model`, whose buffers are empty. A + /// tree of another structure (other union arms, optionals or slice + /// lengths: `mem.pairBufferized`) is `InvalidBinding`, logged with the + /// field path, before any buffer is written. `start` empties them + /// again, and each stays empty until its load writes it whole. So + /// after any error each buffer is empty or holds shards for the caller + /// to free: `mem.deinitBufferized` frees the tree either way. `buffers` + /// and `store` are borrowed until `wait` or `cancel` returns; do not end + /// `progress` before then. `start` adds one item to `progress` for each + /// tensor the loader fills, a composite one once, and for each program + /// input; the backend completes them. + /// Several loads may be started before one `wait`, from one store or + /// several. The first `start` waits for `init`'s task, and fails with + /// its error. Each `start` then gives the device memory freed before it + /// back to the devices (`Platform.releaseFreedDeviceMemory`). + pub fn start( + self: *Loader, + model: anytype, + buffers: *Bufferized(@TypeOf(model.*)), + store: *const TensorStore, + progress: std.Progress.Node, + ) !void { + if (self.failure) |err| return err; + self.startModel(model, buffers, store, progress) catch |err| { + self.fail(err); + return err; + }; + } + + fn startModel( + self: *Loader, + model: anytype, + buffers: *Bufferized(@TypeOf(model.*)), + store: *const TensorStore, + progress: std.Progress.Node, + ) !void { + try self.awaitSetup(); + // Give back what was freed before this load (compile scratch, the + // set-up's calibration buffers), so that admission's room does not + // count it and the load's peak does not include it. + self.platform.releaseFreedDeviceMemory(); + // The load's time starts once the set-up is done and the freed + // memory given back. + self.stats.begin(self.io); + var prepared = try prepareModel(self.allocator, self.platform, self.backend.?, store, model, buffers); + errdefer prepared.arena.deinit(); + var items = prepared.specs.len; + for (prepared.computed) |item| items += item.compute.inputs.len; + progress.increaseEstimatedTotalItems(items); + // Queued first: their programs compile while the rest is read. + try self.queueGroups(store, prepared.computed, progress); + if (prepared.specs.len == 0) { + prepared.arena.deinit(); + return; + } + try self.submit(prepared.specs, prepared.arena, &.{}, @splat(0), progress); + } + + /// Publishes the queued program groups, oldest first, each once its + /// programs compiled and the devices have room for it (`admit`), and + /// retires every submission in publish order, running a group's + /// programs on this task once its inputs landed. Returns once every + /// started load is in its buffers, with the freed device memory, its + /// programs' inputs and temporaries among it, given back to the devices + /// (`Platform.releaseFreedDeviceMemory`), and logs one info line: the + /// bytes loaded, the time since the first `start` and their rate, then + /// what the programs computed and how long publishing waited for their + /// compiles. A program that fails to compile or run fails it; one that + /// does not return its tensor's shape, placed alike, is + /// `InvalidBinding`; a group that does not fit a device fails it with + /// the device's error. Idempotent: with nothing started it returns at + /// once and logs nothing. + pub fn wait(self: *Loader) !void { + if (self.failure) |err| return err; + const started = self.loadsStarted(); + self.drain() catch |err| { + self.fail(err); + return err; + }; + self.releasePrograms(); + if (started) self.platform.releaseFreedDeviceMemory(); + } + + fn drain(self: *Loader) !void { + var groups: usize = 0; + var tensors: usize = 0; + while (self.queued.popFront()) |group| { + tensors += group.calls.len; + try self.publish(group); + groups += 1; + } + while (self.pending.len != 0) try self.retireOldest(true); + self.stats.end(self.io, tensors, self.programs.items.len, groups); + } + + /// Records `err` unless the loader failed already, and gives up on + /// everything started (`cancel`). The one writer of `failure`. + fn fail(self: *Loader, err: anyerror) void { + if (self.failure == null) self.failure = err; + while (self.queued.popFront()) |group| group.deinit(self.allocator); + // Nothing still pending can succeed. + if (self.pending.len != 0) self.backend.?.abort(); + while (self.pending.len != 0) self.retireOldest(false) catch {}; + self.releasePrograms(); + } + + /// Queues the tensors of `computed`, in model order, in groups of at + /// most `program_group_bytes` of outputs, one tensor at least. + fn queueGroups(self: *Loader, store: *const TensorStore, computed: []const Computed, progress: std.Progress.Node) !void { + var first: usize = 0; + while (first < computed.len) { + var end = first + 1; + var bytes = computed[first].shape.byteSize(); + while (end < computed.len and bytes + computed[end].shape.byteSize() <= self.program_group_bytes) : (end += 1) { + bytes += computed[end].shape.byteSize(); + } + try self.queued.ensureUnusedCapacity(self.allocator, 1); + self.queued.pushBackAssumeCapacity(try self.prepareGroup(store, computed[first..end], progress)); + first = end; + } + } + + /// The group computing `computed`: each tensor's program, the input + /// shells it runs over and the specs that fill them, one per input, + /// checked as `start` checks every spec (`prepareSpec`) before the + /// program's compile starts, which packs their shapes. + fn prepareGroup(self: *Loader, store: *const TensorStore, computed: []const Computed, progress: std.Progress.Node) PrepareError!Group { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + errdefer arena.deinit(); + const calls = try self.allocator.alloc(Call, computed.len); + var prepared: usize = 0; + errdefer { + for (calls[0..prepared]) |*call| call.deinit(self.allocator); + self.allocator.free(calls); + } + var specs: std.ArrayList(LoadSpec) = .empty; + for (computed, calls) |item, *call| { + const inputs = try self.allocator.alloc(Buffer, item.compute.inputs.len); + errdefer self.allocator.free(inputs); + for (item.compute.inputs, inputs, 0..) |source, *input, index| { + input.* = .{ ._platform = self.platform, ._shape = source.shape, ._shards = .empty }; + const target: Target = .{ .path = item.path, .shape = item.shape, .input = .{ .index = index, .shape = source.shape } }; + const spec = try prepareSpec(arena.allocator(), self.platform, self.backend.?, target, source, .default, input); + try specs.append(arena.allocator(), spec); + } + const compiled = try self.compiledProgram(store, item.compute, item.shape); + call.* = .{ .path = try arena.allocator().dupe(u8, item.path), .program = compiled, .inputs = inputs, .output = item.output }; + prepared += 1; + } + return .{ .calls = calls, .specs = specs.items, .arena = arena, .progress = progress }; + } + + /// The compiled program computing a tensor of `output` from `compute`: + /// one started already for an equal program over the same placed + /// shapes, else a compile started now. + fn compiledProgram(self: *Loader, store: *const TensorStore, compute: TensorStore.Binding.Compute, output: Shape) std.mem.Allocator.Error!*CompiledProgram { + for (self.programs.items) |compiled| { + if (compiled.compute.sameExecutable(compiled.output, compute, output)) return compiled; + } + try self.programs.ensureUnusedCapacity(self.allocator, 1); + const compiled = try CompiledProgram.start(self.allocator, self.io, self.platform, store.meshes, compute, output); + self.programs.appendAssumeCapacity(compiled); + return compiled; + } + + fn releasePrograms(self: *Loader) void { + for (self.programs.items) |compiled| compiled.destroy(self.allocator, self.io); + self.programs.clearRetainingCapacity(); + } + + /// Publishes `group` once its programs compiled and the devices have + /// room for it: its calls and its specs' arena move to the pending + /// submission, or are freed when this fails. + fn publish(self: *Loader, group: Group) !void { + errdefer group.deinit(self.allocator); + const started: std.Io.Timestamp = .now(self.io, .awake); + for (group.calls) |call| try call.program.await(self.io, self.platform); + self.stats.compile_wait_ns +|= @intCast(started.untilNow(self.io, .awake).toNanoseconds()); + const execution = try self.admit(group.calls); + try self.submit(group.specs, group.arena, group.calls, execution, group.progress); + } + + /// Retires pending submissions, oldest first, until the group of + /// `calls` fits the measured room or no program group is pending: a + /// bulk load frees nothing. Once nothing can be retired for it, the + /// room is read again after the freed device memory is given back + /// (`Platform.releaseFreedDeviceMemory`), and the group is admitted + /// whatever its size (`Stats.beyond_room`). Without memory measurement + /// one program group is in flight at a time. Returns the group's output + /// and temporary bytes per device, zero while unmeasured. + fn admit(self: *Loader, calls: []const Call) !DeviceBytes { + var inputs: DeviceBytes = @splat(0); + var execution: DeviceBytes = @splat(0); + var room: ?DeviceBytes = null; + if (self.measured) |direct| { + var temporaries: DeviceBytes = @splat(0); + for (calls) |call| { + const exe = &call.program.exe.?; + for (exe.input_shapes) |shape| addPlacementBytes(&inputs, self.platform, shape); + // The output's memory space is the program's to choose; it is + // charged to the devices like the default space. + addPlacementBytes(&execution, self.platform, exe.output_shapes[0]); + // The group's programs run one after the other, so only the + // largest temporaries are live at once. + const temp = self.compiledTempBytes(exe); + const output_mesh = self.platform.placedShape(exe.output_shapes[0])._sharding.mesh.?; + for (output_mesh.devicesInCanonicalOrder()) |device| temporaries[device.id] = @max(temporaries[device.id], temp); + } + for (&execution, temporaries) |*bytes, temp| bytes.* +|= temp; + room = self.readRoom(direct); + } + var retired: usize = 0; + while (room == null or !self.fits(&room.?, &inputs, &execution)) : (retired += 1) { + if (!self.programGroupPending()) { + // Nothing is left to retire, but the retired groups' inputs + // may still count as in use: read the room again once the + // devices have them back. The group then goes, beyond the + // room if the room is still short. + if (room != null) { + self.platform.releaseFreedDeviceMemory(); + room = self.readRoom(self.measured.?); + } + break; + } + try self.retireOldest(true); + if (self.measured) |direct| room = self.readRoom(direct); + } + const beyond_room = room != null and !self.fits(&room.?, &inputs, &execution); + self.stats.admission_retires += retired; + if (beyond_room) self.stats.beyond_room += 1; + if (retired != 0 or beyond_room) { + // Unmeasured, the group's bytes are not computed: only the retires are logged. + if (self.measured != null) { + const n = self.platform.devices.len; + // A device that stopped reporting its limit shows no room. + const rooms: DeviceBytes = room orelse @splat(0); + const pending = self.pendingExecution(); + // The device the fit is decided on: the short one when the + // group goes beyond the room. + const device = admission.tightest(rooms[0..n], pending[0..n], inputs[0..n], execution[0..n]); + load_log.debug("execute admission: retired={d} before submission {d}: measured=true, beyond_room={}, device {d} room={Bi:.2} pending_execution={Bi:.2} inputs={Bi:.2} execution={Bi:.2}", .{ + retired, + self.stats.submissions, + beyond_room, + device, + rooms[device], + pending[device], + inputs[device], + execution[device], + }); + } else { + load_log.debug("execute admission: retired={d} before submission {d}: measured=false", .{ retired, self.stats.submissions }); + } + } + return execution; + } + + /// Whether a group of `inputs` and `execution` fits beside the pending + /// program groups in `room`, on every device. + fn fits(self: *const Loader, room: *const DeviceBytes, inputs: *const DeviceBytes, execution: *const DeviceBytes) bool { + const n = self.platform.devices.len; + const pending = self.pendingExecution(); + return admission.admits(room[0..n], pending[0..n], inputs[0..n], execution[0..n]); + } + + /// Output and temporary bytes of the pending program groups, per + /// device: what retiring them will take before it frees anything. + fn pendingExecution(self: *const Loader) DeviceBytes { + var sum: DeviceBytes = @splat(0); + var it = self.pending.iterator(); + while (it.nextPtr()) |submission| { + for (&sum, submission.execution) |*bytes, more| bytes.* +|= more; + } + return sum; + } + + /// Whether a program group is pending: only retiring one frees room + /// for the next (a bulk load's buffers stay). + fn programGroupPending(self: *const Loader) bool { + var it = self.pending.iterator(); + while (it.nextPtr()) |submission| { + if (submission.calls.len != 0) return true; + } + return false; + } + + /// The room each device reports now; null when one reports no memory + /// limit. + fn readRoom(self: *Loader, direct: *const direct_loader.Loader) ?DeviceBytes { + const n = self.platform.devices.len; + const unallocated = direct.unallocatedBytes(); + var room: DeviceBytes = undefined; + for (room[0..n], self.platform.devices, unallocated[0..n]) |*out, device, device_unallocated| { + const reported = device.memoryStats(); + out.* = admission.room(reported.bytes_limit, reported.bytes_in_use, device_unallocated, admission.reserve_bytes) orelse return null; + self.stats.min_room_seen = @min(self.stats.min_room_seen orelse out.*, out.*); + } + return room; + } + + /// Temporaries the executable takes on each of its devices, from its + /// compiled memory stats (per partition, i.e. per device). Zero when + /// the plugin cannot answer. + fn compiledTempBytes(self: *Loader, exe: *const Exe) u64 { + const api = self.platform.pjrt_api; + const pjrt_executable = exe.exe.executable(api) catch |err| return self.tempUnavailable(err); + defer pjrt_executable.deinit(api); + const stats = pjrt_executable.getCompiledMemoryStats(api) catch |err| return self.tempUnavailable(err); + return stats.temp_size_in_bytes; + } + + fn tempUnavailable(self: *Loader, err: pjrt.ApiError) u64 { + if (!self.stats.temp_unavailable_logged) { + self.stats.temp_unavailable_logged = true; + load_log.debug("executable memory stats unavailable ({s}): temporaries counted as zero", .{@errorName(err)}); + } + return 0; + } + + /// One submission over `specs`, which live in `arena`. Takes `arena` and + /// `calls` once the submission is published; on failure the caller still + /// owns them. `execution` is a program group's output and temporary + /// bytes per device (`admit`), zero for a bulk load. + fn submit( + self: *Loader, + specs: []const LoadSpec, + arena: std.heap.ArenaAllocator, + calls: []Call, + execution: DeviceBytes, + progress: std.Progress.Node, + ) !void { + // One spec per destination, however many parts fill it: its bytes + // count once. + var logical_bytes: usize = 0; + for (specs) |spec| logical_bytes += spec.shape.byteSize(); + try self.pending.ensureUnusedCapacity(self.allocator, 1); + const submission = try self.backend.?.submit(specs, progress); + self.pending.pushBackAssumeCapacity(.{ + .submission = submission, + .arena = arena, + .calls = calls, + .execution = execution, + .logical_bytes = logical_bytes, + }); + self.stats.submissions += 1; + } + + /// Retires the oldest pending submission: waits for its reads and + /// transfers, runs its programs when `execute`, frees its inputs either + /// way and counts its bytes once they ran. + fn retireOldest(self: *Loader, execute: bool) !void { + var oldest = self.pending.popFront().?; + defer self.release(&oldest); + try self.backend.?.await(oldest.submission); + if (!execute) return; + for (oldest.calls) |*call| { + call.execute(self.allocator, self.io) catch |err| { + const target: Target = .{ .path = call.path, .shape = call.program.output }; + load_log.failure(err, "run program {s} computing {f}: inputs={d}", .{ call.program.name(), target, call.inputs.len }); + return err; + }; + } + self.stats.bytes_loaded += oldest.logical_bytes; + } + + fn release(self: *Loader, oldest: *PendingSubmission) void { + oldest.arena.deinit(); + for (oldest.calls) |*call| call.deinit(self.allocator); + self.allocator.free(oldest.calls); + } +}; + +/// The loader's counters: read only by logs and tests; nothing here +/// decides. +const Stats = struct { + /// When the first `start` since the last `wait` ran: `wait` logs the + /// load's time from there. + started_at: ?std.Io.Timestamp = null, + /// Logical bytes of the submissions retired with execution since that + /// `start`: each destination's shape once, so a composite tensor counts + /// its padding and not its sources one by one, and a program group + /// counts its inputs. + bytes_loaded: usize = 0, + /// How long publishing waited for compiles since the last `wait`. + compile_wait_ns: u64 = 0, + /// Submissions published since `init`. + submissions: usize = 0, + /// Submissions retired to admit a program group, bulk loads included. + admission_retires: usize = 0, + /// The least room a device reported since `init`; null while nothing + /// was measured. + min_room_seen: ?u64 = null, + /// Program groups admitted beyond the measured room since `init`: + /// nothing was left to retire for them. + beyond_room: usize = 0, + /// Whether `tempUnavailable` logged that the plugin gives no compiled + /// memory stats: once per loader. + temp_unavailable_logged: bool = false, + + /// Times and counts a load from the first `start` since the last + /// `wait`; later starts add to it. + fn begin(self: *Stats, io: std.Io) void { + if (self.started_at != null) return; + self.started_at = .now(io, .awake); + self.bytes_loaded = 0; + } + + /// Logs `wait`'s info line, unless nothing was started, and ends the + /// load's timing. `programs` programs computed `tensors` tensors in + /// `groups` groups. + fn end(self: *Stats, io: std.Io, tensors: usize, programs: usize, groups: usize) void { + const started_at = self.started_at orelse return; + const took = started_at.untilNow(io, .awake); + const seconds = @as(f64, @floatFromInt(@max(took.nanoseconds, 1))) / std.time.ns_per_s; + const rate: u64 = @intFromFloat(@as(f64, @floatFromInt(self.bytes_loaded)) / seconds); + if (groups == 0) { + load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ self.bytes_loaded, took, rate }); + } else { + load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s], computed {d} tensors with {d} programs in {d} groups, {d} ms waiting for compiles", .{ + self.bytes_loaded, + took, + rate, + tensors, + programs, + groups, + self.compile_wait_ns / std.time.ns_per_ms, + }); + } + self.started_at = null; + self.compile_wait_ns = 0; + } + + /// `deinit`'s debug line. `measured`: whether admission measured the + /// room (`Loader.measured`), printed as `memory_supported`. + fn logAdmission(self: *const Stats, measured: bool) void { + if (self.submissions == 0) return; + load_log.debug("loader admission: submissions={d}, execute_admission_retires={d}, beyond_room={d}, memory_supported={}, min_room_seen={Bi:.2}, reserve={Bi:.2}", .{ + self.submissions, + self.admission_retires, + self.beyond_room, + measured, + self.min_room_seen orelse 0, + admission.reserve_bytes, + }); + } +}; + +/// One published submission: what to wait for, what to run afterwards and +/// what its accounting owes. +const PendingSubmission = struct { + submission: backend.Submission, + /// The specs, which the backend reads until the submission is awaited + /// (`backend.LoadSpec`). + arena: std.heap.ArenaAllocator, + /// Run in order once the reads are done; freed with their inputs when + /// the submission is retired. Empty for a bulk load. + calls: []Call, + /// Output and temporary bytes per device of a program group, while + /// admission measures (`admit`); zero otherwise and for a bulk load. + execution: DeviceBytes, + logical_bytes: usize, +}; + +/// Program tensors whose inputs `wait` publishes as one submission. +const Group = struct { + calls: []Call, + /// One per program input, filling the calls' input shells; they and + /// their parts live in `arena`. + specs: []LoadSpec, + arena: std.heap.ArenaAllocator, + progress: std.Progress.Node, + + fn deinit(self: Group, allocator: std.mem.Allocator) void { + for (self.calls) |*call| call.deinit(allocator); + allocator.free(self.calls); + self.arena.deinit(); + } +}; + +/// A program compiling, then compiled. +const CompiledProgram = struct { + compute: TensorStore.Binding.Compute, + output: Shape, + future: std.Io.Future(Compiler.Error!Exe), + /// Set by the first `await`. + exe: ?Exe = null, + + fn start( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + meshes: []const *const Sharding.Mesh, + compute: TensorStore.Binding.Compute, + output: Shape, + ) std.mem.Allocator.Error!*CompiledProgram { + const self = try allocator.create(CompiledProgram); + // Concurrent, as `io.async` may run the compile right here on a busy + // threaded io, ahead of the reads `start` publishes after it. + const args = .{ compute, allocator, io, platform, meshes, output }; + const compile = TensorStore.Binding.Compute.compile; + self.* = .{ + .compute = compute, + .output = output, + .future = io.concurrent(compile, args) catch io.async(compile, args), + }; + return self; + } + + /// Waits for the compile the first time, then checks that the + /// executable computes one tensor of `output`, placed alike + /// (`checkOutput`). + fn await(self: *CompiledProgram, io: std.Io, platform: *const Platform) !void { + if (self.exe != null) return; + self.exe = self.future.await(io) catch |err| { + load_log.failure(err, "compile program {s} computing {f}", .{ self.name(), self.output }); + return err; + }; + try checkOutput(platform, self.name(), self.output, &self.exe.?); + } + + fn name(self: *const CompiledProgram) []const u8 { + return self.compute.program.name(); + } + + fn destroy(self: *CompiledProgram, allocator: std.mem.Allocator, io: std.Io) void { + if (self.exe) |*exe| { + exe.deinit(); + } else if (self.future.cancel(io)) |exe| { + exe.deinit(); + } else |_| {} + allocator.destroy(self); + } +}; + +/// One tensor of a program group, a pending `Exe.call`: its program, the +/// input shells the group's submission fills, and the model's buffer it +/// computes. +const Call = struct { + /// The tensor's field path, in the group's arena: `prepareModel`'s + /// arena may be freed before the call runs. + path: []const u8, + program: *CompiledProgram, + inputs: []Buffer, + output: *Buffer, + + /// Runs the program's executable, awaited when the group was + /// published, over the loaded inputs on the calling task. + /// `Exe.call` leaves every input handle to the caller, donated ones + /// included (their contents are gone), so `deinit` frees them all, + /// exactly once; a runner would destroy the donated handles itself. + /// A failed execution, an input whose transfer failed included, is + /// returned and leaves `output` unwritten. + fn execute(self: *const Call, allocator: std.mem.Allocator, io: std.Io) !void { + const exe = &self.program.exe.?; + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + args.set(.{self.inputs}); + try exe.call(io, args, &results, .{ .wait = true }); + self.output.* = results.get(Buffer); + } + + /// Frees the inputs, donated or not; a shell the loader never wrote owns + /// no shards. + fn deinit(self: *Call, allocator: std.mem.Allocator) void { + for (self.inputs) |*input| input.deinit(); + allocator.free(self.inputs); + } +}; + +/// The arithmetic behind program admission: what the devices have left, +/// and whether the next group fits beside the ones still pending. +/// Everything is per device (index = `platform.devices` index) in bytes; +/// nothing here touches a device, so the rules are tested without one. +const admission = struct { + /// Allocator slack per device. BFC rounds every allocation up to 256 bytes; + /// a 64-pack, 16-source, 8-device load is about 1e5 shards, under 32 MiB of + /// rounding, and nothing else runs on the devices during a load. + const reserve_bytes: u64 = 64 << 20; + + /// Bytes the loader may still take on one device: the limit (null when the + /// plugin reports none, as CPU does) minus what is in use, minus its own + /// bytes published but not allocated yet (`unallocated`), minus the + /// reserve. + fn room(limit: ?u64, in_use: u64, unallocated: u64, reserve: u64) ?u64 { + return (limit orelse return null) -| in_use -| unallocated -| reserve; + } + + /// Whether one submission's `inputs` (placements, counted here until they + /// are published) and `execution` (output placement plus the executable's + /// temporaries) fit beside the pending executions on every device. + fn admits(rooms: []const u64, pending: []const u64, inputs: []const u64, execution: []const u64) bool { + for (rooms, pending, inputs, execution) |r, p, i, e| { + if (p +| i +| e > r) return false; + } + return true; + } + + /// The device with the least room left once one submission's `inputs` + /// and `execution` sit beside the pending executions: one that `admits` + /// fails on when the submission does not fit. `rooms` is not empty. + fn tightest(rooms: []const u64, pending: []const u64, inputs: []const u64, execution: []const u64) usize { + var device: usize = 0; + var least: i128 = std.math.maxInt(i128); + for (rooms, pending, inputs, execution, 0..) |r, p, i, e, index| { + const left = @as(i128, r) - (p +| i +| e); + if (left < least) { + least = left; + device = index; + } + } + return device; + } +}; + +test "room subtracts use, unallocated bytes and the reserve, saturating" { + try std.testing.expectEqual(null, admission.room(null, 0, 0, 0)); + try std.testing.expectEqual(55, admission.room(100, 30, 10, 5).?); + // More in use than the limit admits: no room, no underflow. + try std.testing.expectEqual(0, admission.room(100, 130, 0, 0).?); + try std.testing.expectEqual(0, admission.room(100, 90, 0, 20).?); +} + +test "admits needs room on every device" { + try std.testing.expect(admission.admits(&.{ 40, 40 }, &.{ 20, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); + // The second device is one byte short. + try std.testing.expect(!admission.admits(&.{ 40, 34 }, &.{ 20, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); + try std.testing.expect(admission.admits(&.{}, &.{}, &.{}, &.{})); +} + +test "tightest names the device short of room, not the first" { + // Device 0 has less room but holds the group; device 1 is one byte short. + try std.testing.expectEqual(1, admission.tightest(&.{ 30, 34 }, &.{ 0, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); + // Both are short: the one short by more, though it has more room. + try std.testing.expectEqual(1, admission.tightest(&.{ 10, 12 }, &.{ 0, 0 }, &.{ 10, 20 }, &.{ 5, 10 })); + // A need that saturates at maxInt(u64) is short by the most. + try std.testing.expectEqual(1, admission.tightest(&.{ 10, 20 }, &.{ 0, std.math.maxInt(u64) }, &.{ 20, 1 }, &.{ 0, 0 })); + // Equally short: the first. + try std.testing.expectEqual(0, admission.tightest(&.{ 10, 10 }, &.{ 0, 0 }, &.{ 12, 12 }, &.{ 0, 0 })); +} + +/// Adds the bytes `shape` occupies per device: its box on each device it +/// is placed on, as the direct backend's `countSealed` counts them; an +/// empty shape takes none. `shape` places on the platform's devices +/// (`checkPlacement`). +fn addPlacementBytes(out: *DeviceBytes, platform: *const Platform, shape: Shape) void { + if (shape.byteSize() == 0) return; + const grid: ShardGrid = .init(platform, shape); + for (grid.devices) |device| out[device.id] += grid.box_bytes; +} + +/// A tensor a program computes, at `path` in the model, the buffer it +/// computes into, and its program and inputs. +const Computed = struct { + path: []const u8, + shape: Shape, + output: *Buffer, + compute: TensorStore.Binding.Compute, +}; + +/// A model as `start` takes it: one spec per tensor the loader fills, and +/// the tensors programs compute, all in `arena` with the specs' parts and +/// the tensors' field paths. +const PreparedModel = struct { + arena: std.heap.ArenaAllocator, + specs: []const LoadSpec, + computed: []const Computed, +}; + +/// Pairs each tensor of `model` with its buffer and binding: a tensor the +/// loader fills takes one spec (`prepareSpec`), a tensor a program computes +/// is placed like any other (`checkPlacement`) and computed. +fn prepareModel( + allocator: std.mem.Allocator, + platform: *const Platform, + backend_: Backend, + store: *const TensorStore, + model: anytype, + buffers: *Bufferized(@TypeOf(model.*)), +) PrepareError!PreparedModel { + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + // Tensors and buffers pair by structure, before any buffer is written: a + // buffer tree shaped otherwise is refused whole. + var report_buffer: [256]u8 = undefined; + var report: std.Io.Writer = .fixed(&report_buffer); + const pairs = mem.pairBufferized(arena.allocator(), model, buffers, &report) catch |err| switch (err) { + error.Mismatch => { + load_log.err("buffers shaped unlike the model at model{s}; bufferize them from this model", .{report.buffered()}); + return error.InvalidBinding; + }, + error.OutOfMemory => return error.OutOfMemory, + }; + // Empty until a backend or a program writes it whole, so the caller can + // free the tree whatever fails. + for (pairs) |pair| pair.buffer.* = .{ ._platform = platform, ._shape = pair.tensor.shape(), ._shards = .empty }; + + var specs: std.ArrayList(LoadSpec) = try .initCapacity(arena.allocator(), pairs.len); + var computed: std.ArrayList(Computed) = try .initCapacity(arena.allocator(), pairs.len); + for (pairs) |pair| { + const tensor = pair.tensor; + const target: Target = .{ .path = pair.path, .shape = tensor.shape() }; + const binding = store.bindingOf(tensor.id) orelse { + load_log.err("{f} has no checkpoint binding", .{target}); + return error.InvalidBinding; + }; + switch (binding) { + .load => |load| specs.appendAssumeCapacity(try prepareSpec(arena.allocator(), platform, backend_, target, load.source, load.memory, pair.buffer)), + .compute => |compute| { + try checkPlacement(platform, target, compute.program.name()); + computed.appendAssumeCapacity(.{ .path = pair.path, .shape = tensor.shape(), .output = pair.buffer, .compute = compute }); + }, + } + } + return .{ .arena = arena, .specs = specs.items, .computed = computed.items }; +} + +/// A strided part with few runs, or with large ones, expands into one part +/// per run, each read and transferred on its own: at most `max_part_runs` +/// runs, or runs of at least `large_run_bytes`. A spec with any other +/// strided part goes to the backend whole, as strided parts +/// (`backend.Lattice`), which the direct backend reads by destination +/// ranges. +const max_part_runs = 64; +const large_run_bytes = 1 << 20; + +fn expandable(lattice: backend.Lattice) bool { + return lattice.dims.len == 0 or lattice.runs() <= max_part_runs or lattice.width >= large_run_bytes; +} + +/// What a spec fills, as the logs name it: the model's tensor at `path` +/// (`mem.Pair.path`) of `shape`, or `input` of the program computing it. +const Target = struct { + path: []const u8, + shape: Shape, + input: ?Input = null, + + const Input = struct { index: usize, shape: Shape }; + + /// The shape the spec fills: the input's, else the tensor's. + fn destination(self: Target) Shape { + return if (self.input) |input| input.shape else self.shape; + } + + pub fn format(self: Target, writer: *std.Io.Writer) !void { + if (self.input) |input| try writer.print("program input {d} {f} of ", .{ input.index, input.shape }); + try writer.print("model{s} {f}", .{ self.path, self.shape }); + } +}; + +/// The spec filling `output`, the buffer of `target`, from `source` into +/// `memory`, its parts in `arena`: the source lowered to bytes +/// (`Source.Placed.lower`), expanded into its runs unless a part does not +/// expand, in destination order. The one check of what the backends trust +/// (`backend.LoadSpec`, `Backend.stridedRead`); a rule it breaks is +/// `InvalidBinding`, logged with the target, and with the key its source +/// reads once the source is lowered. +fn prepareSpec( + arena: std.mem.Allocator, + platform: *const Platform, + backend_: Backend, + target: Target, + source: TensorStore.Source.Placed, + memory: Memory, + output: *Buffer, +) PrepareError!LoadSpec { + const shape = target.destination(); + const lowered = source.lower(arena) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.SubBytePosition => { + load_log.err("{f}: its source {f} reads a run that starts or ends inside a byte, which the loader cannot place", .{ target, source.shape }); + return error.InvalidBinding; + }, + }; + if (lowered.len == 0) { + load_log.err("{f} reads no checkpoint bytes", .{target}); + return error.InvalidBinding; + } + const key = lowered[0].source.name; + var expands = true; + var runs: u64 = 0; + for (lowered) |part| { + expands = expands and expandable(part.lattice); + runs += part.lattice.runs(); + } + try checkPlacement(platform, target, key); + if (!backend.supportsMemory(platform, memory)) { + load_log.err("{f} from {s} is bound to {t} memory, which {t} cannot take", .{ target, key, memory, platform.target }); + return error.InvalidBinding; + } + if (!expands and memory.isHost()) { + load_log.err("{f} from {s}: strided parts into {t} memory, which this loader does not assemble", .{ target, key, memory }); + return error.InvalidBinding; + } + const parts = if (!expands) lowered else expanded: { + if (runs == lowered.len) break :expanded lowered; + const expanded = try arena.alloc(LoadPart, @intCast(runs)); + var next: usize = 0; + for (lowered) |part| { + var cursor: backend.Lattice.Cursor = .init(part.lattice, 0, part.lattice.runs()); + while (cursor.next()) |run| : (next += 1) { + expanded[next] = .{ .source = part.source, .lattice = .{ .src_offset = run.src, .dst_offset = run.dst, .width = part.lattice.width } }; + } + } + break :expanded expanded; + }; + std.mem.sort(LoadPart, parts, {}, struct { + fn lessThan(_: void, lhs: LoadPart, rhs: LoadPart) bool { + return lhs.lattice.dst_offset < rhs.lattice.dst_offset; + } + }.lessThan); + if (!backend.checkParts(parts, shape.byteSize())) { + load_log.err("{f} from {s}: its parts leave their sources or its {d} bytes, pass the largest file offset, or come out of order", .{ target, key, shape.byteSize() }); + return error.InvalidBinding; + } + // After `checkParts`, which bounds the offsets `stridedRead` adds. + for (parts) |part| { + const strided = backend_.stridedRead(part); + if (strided.read > strided.limit) { + load_log.err("{f} from {s}: a strided part spans {d} source bytes, more than one read of {d}", .{ target, key, strided.read, strided.limit }); + return error.InvalidBinding; + } + } + return .{ .shape = shape, .memory = memory, .output = output, .parts = parts }; +} + +/// Checks that `target`, from `from` (its source's key or its program), +/// places on `platform`'s devices, in any memory space: a sub-byte last +/// dimension packs into whole bytes (`Shape.packedShape`), its mesh +/// partitions divide its dimensions, and every device of its mesh is one of +/// the platform's. +fn checkPlacement(platform: *const Platform, target: Target, from: []const u8) error{InvalidBinding}!void { + const shape = target.destination(); + const bits = shape.dtype().bitSizeOf(); + if (bits < 8 and (shape.rank() == 0 or @rem(shape.dim(-1), 8 / bits) != 0)) { + load_log.err("{f} from {s}: its last dimension does not pack into whole bytes", .{ target, from }); + return error.InvalidBinding; + } + const placed = platform.placedShape(shape).packedShape(); + const placement = Sharding.Placement.init(&placed) catch { + load_log.err("{f} from {s}: its dimensions are not divisible by its mesh partitions", .{ target, from }); + return error.InvalidBinding; + }; + for (placement.mesh().?.devicesInCanonicalOrder()) |device| { + if (device.id >= platform.devices.len) { + load_log.err("{f} from {s} is placed on device {d}, but {t} has {d}", .{ target, from, device.id, platform.target, platform.devices.len }); + return error.InvalidBinding; + } + } +} + +/// `opts` as the backends take them, the checkpoint resolved to the read +/// hints of the VFS backend serving it and the read width to its default, +/// and borrowed when it is local and may read direct +/// (`backend.Options.checkpoint`). Every option is checked here, once. +fn backendOptions(io: std.Io, opts: Loader.Options) error{InvalidOptions}!backend.Options { + const hints: VFS.ReadHints = hints: { + const path = opts.checkpoint orelse break :hints .local; + const vfs = VFS.fromIo(io) orelse break :hints .local; + break :hints vfs.readHints(path) catch |err| { + load_log.err("invalid loader options: checkpoint={s}: {s}", .{ path, @errorName(err) }); + return error.InvalidOptions; + }; + }; + if (opts.read_parallelism) |width| { + if (width == 0 or width > Loader.max_read_parallelism) { + load_log.err("invalid loader options: read_parallelism={d}, expected 1..{d}: InvalidOptions", .{ width, Loader.max_read_parallelism }); + return error.InvalidOptions; + } + } + const max_request = backend.max_read_request_size; + if (hints.read_chunk_size == 0 or hints.read_chunk_size > max_request) { + load_log.err("invalid loader options: checkpoint={?s}, read_chunk_size={d}, expected 1..{d}: InvalidOptions", .{ opts.checkpoint, hints.read_chunk_size, max_request }); + return error.InvalidOptions; + } + var options: backend.Options = .{ + .read_parallelism = opts.read_parallelism orelse if (hints.high_latency) + Loader.Options.high_latency_read_parallelism + else + Loader.Options.local_read_parallelism, + .read_hints = hints, + .dma_block_size = opts.dma_block_size, + .direct_io = opts.direct_io, + }; + // Checked for every block calibration may select. + const alignment = backend.readAlignment(io, opts.direct_io); + for (options.dmaBlockSizes()) |block_size| { + if (backend.requestSize(hints.read_chunk_size, block_size, alignment) == null) { + load_log.err("invalid loader options: checkpoint={?s}, read_chunk_size={d}, dma_block_size={d}, alignment={d}, max_request={d}: no request size fits: InvalidOptions", .{ opts.checkpoint, hints.read_chunk_size, block_size, alignment, max_request }); + return error.InvalidOptions; + } + } + // A nonzero alignment means a VFS that may read a local file direct. + // `isLocal` opens nothing. + const path = opts.checkpoint orelse return options; + if (alignment != 0 and VFS.fromIo(io).?.isLocal(path)) options.checkpoint = path; + return options; +} + +/// A program must compute one tensor of `expected`, placed on the same +/// devices, each holding the same slice. The comparison is per device, not +/// by mesh pointer and partition, so `.open` and `.replicated`, or a null +/// mesh and the replicated mesh, agree when they place alike. `expected` +/// places on the platform's devices (`checkPlacement`). +fn checkOutput(platform: *const Platform, name: []const u8, expected: Shape, exe: *const Exe) error{InvalidBinding}!void { + if (exe.output_shapes.len != 1 or !expected.eql(exe.output_shapes[0])) { + load_log.err("program {s} computing {f} returns {d} outputs, the first {f}", .{ name, expected, exe.output_shapes.len, if (exe.output_shapes.len != 0) exe.output_shapes[0] else expected }); + return error.InvalidBinding; + } + const actual = exe.output_shapes[0]; + const want_shape = platform.placedShape(expected).packedShape(); + const want = Sharding.Placement.init(&want_shape) catch |err| + std.debug.panic("loader: {f} does not place, though checked: {s}", .{ expected, @errorName(err) }); + const got_shape = platform.placedShape(actual).packedShape(); + const got = Sharding.Placement.init(&got_shape) catch |err| { + load_log.err("program {s}: placement of its output {f}: {s}", .{ name, actual, @errorName(err) }); + return error.InvalidBinding; + }; + const want_devices = want.mesh().?.devicesInCanonicalOrder(); + const got_devices = got.mesh().?.devicesInCanonicalOrder(); + if (want_devices.len != got_devices.len) { + load_log.err("program {s} places its output {f} on {d} devices, the tensor {f} on {d}", .{ name, actual, got_devices.len, expected, want_devices.len }); + return error.InvalidBinding; + } + for (want_devices, got_devices, 0..) |w, g, device_index| { + const want_slices = want.slices(device_index, want_shape); + const got_slices = got.slices(device_index, got_shape); + const same_slices = want_slices.len == got_slices.len and for (want_slices.constSlice(), got_slices.constSlice()) |a, b| { + if (a.start != b.start or a.size != b.size) break false; + } else true; + if (w.id != g.id or !same_slices) { + load_log.err("program {s} places its output {f} unlike the tensor {f} on device {d}", .{ name, actual, expected, w.id }); + return error.InvalidBinding; + } + } +} + +/// A four-byte tensor `value`, a four-byte `second`, a `missing` entry in a +/// file that does not exist, an `empty` tensor and a CPU platform. Pinned +/// after `init`. +const LoaderTestFixture = struct { + const contents = [_]u8{ 1, 2, 3, 4 }; + const second_contents = [_]u8{ 5, 6, 7, 8 }; + + tmp: std.testing.TmpDir, + store: TensorStore, + platform: *const Platform, + value: Tensor, + second: Tensor, + missing: Tensor, + empty: Tensor, + + fn init(self: *LoaderTestFixture, allocator: std.mem.Allocator, io: std.Io) !void { + self.tmp = std.testing.tmpDir(.{}); + errdefer self.tmp.cleanup(); + const file = try self.tmp.dir.createFile(io, "weights.bin", .{ .read = true }); + try file.writePositionalAll(io, &(contents ++ second_contents), 0); + var path_buffer: [1024]u8 = undefined; + const path_len = try file.realPath(io, &path_buffer); + file.close(io); + const path = path_buffer[0..path_len]; + var missing_buffer: [1024]u8 = undefined; + const missing_path = try std.fmt.bufPrint(&missing_buffer, "{s}.missing", .{path}); + + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ + .file_uri = path, + .name = "value", + .shape = .init(.{contents.len}, .u8), + .offset = 0, + }); + try registry.registerTensor(.{ + .file_uri = path, + .name = "second", + .shape = .init(.{second_contents.len}, .u8), + .offset = contents.len, + }); + try registry.registerTensor(.{ + .file_uri = missing_path, + .name = "missing", + .shape = .init(.{contents.len}, .u8), + .offset = 0, + }); + try registry.registerTensor(.{ + .file_uri = path, + .name = "empty", + .shape = .init(.{0}, .u8), + .offset = 0, + }); + self.platform = testing.env(); + self.store = .init(allocator, ®istry, .{ .meshes = &.{self.platform.replicated_mesh} }); + self.value = self.store.view().createReplicatedTensor("value", null); + self.second = self.store.view().createReplicatedTensor("second", null); + self.missing = self.store.view().createReplicatedTensor("missing", null); + self.empty = self.store.view().createReplicatedTensor("empty", null); + } + + fn deinit(self: *LoaderTestFixture) void { + self.store.deinit(); + self.tmp.cleanup(); + } + + /// A tensor of `like`'s shape that `program` computes from the + /// checkpoint tensor `name`. + fn computed(self: *LoaderTestFixture, name: []const u8, like: Tensor, program: anytype) Tensor { + const input = self.store.view().source(name, null).withPartitioning(.replicated, .{}); + return TensorStore.Source.bindProgram(&.{input}, like.shape(), program); + } + + /// The program returning its one input. + const Identity = struct { + pub fn forward(_: Identity, _: Shape, inputs: []const Tensor) Tensor { + return inputs[0]; + } + }; + + /// The program adding `value` to every byte of its one input. + const Add = struct { + value: u8, + + pub fn forward(self: Add, _: Shape, inputs: []const Tensor) Tensor { + return inputs[0].add(Tensor.scalar(self.value, .u8)); + } + }; + + /// Whether `loader` failed with `expected` and holds nothing started. + fn expectFailed(loader: *const Loader, expected: anyerror) !void { + try std.testing.expectEqual(expected, loader.failure.?); + try std.testing.expectEqual(0, loader.pending.len + loader.queued.len + loader.programs.items.len); + } +}; + +test "start publishes the bulk at once and wait runs program groups one at a time without memory measurement" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // One program tensor per group. + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); + defer loader.deinit(); + // Exercise serial admission regardless of the platform's memory stats: + // decided once the set-up is done. + try loader.awaitSetup(); + loader.measured = null; + + const Model = struct { value: Tensor, first: Tensor, second: Tensor }; + const model: Model = .{ + .value = fixture.value, + .first = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}), + .second = fixture.computed("second", fixture.second, LoaderTestFixture.Identity{}), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + // The bulk is published; the groups wait for `wait`, their one program + // (the same function and shapes) compiling meanwhile. + try std.testing.expectEqual(1, loader.pending.len); + try std.testing.expectEqual(2, loader.queued.len); + try std.testing.expectEqual(1, loader.programs.items.len); + try loader.wait(); + try std.testing.expectEqual(0, loader.pending.len + loader.queued.len + loader.programs.items.len); + // Serial admission retired the bulk and the first group before it + // published the second. + try std.testing.expectEqual(2, loader.stats.admission_retires); + try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &buffers.first, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &buffers.second, &LoaderTestFixture.second_contents, .default); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); + + // Idempotent: a second wait neither reruns nor recounts. + try loader.wait(); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); + + // The next load counts and times from its own first start. + try std.testing.expectEqual(null, loader.stats.started_at); + var again: Buffer = undefined; + try loader.start(&fixture.second, &again, &fixture.store, .none); + try loader.wait(); + defer again.deinit(); + try std.testing.expectEqual(LoaderTestFixture.second_contents.len, loader.stats.bytes_loaded); + try std.testing.expectEqual(null, loader.stats.started_at); +} + +test "one loader takes loads from different stores before one wait" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var other_store = try testing.storeOf(allocator, &fixture.store.registry, .{ .meshes = &.{fixture.platform.replicated_mesh} }); + defer other_store.deinit(); + const other = other_store.view().createReplicatedTensor("second", null); + + var first: Buffer = undefined; + var second: Buffer = undefined; + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&fixture.value, &first, &fixture.store, .none); + try loader.start(&other, &second, &other_store, .none); + try loader.wait(); + defer first.deinit(); + defer second.deinit(); + try testing.expectShards(allocator, io, &first, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &second, &LoaderTestFixture.second_contents, .default); +} + +test "tensors of one program and equal args share one compile, and one group runs them all" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Add = LoaderTestFixture.Add; + const Model = struct { a: Tensor, b: Tensor, c: Tensor }; + const model: Model = .{ + .a = fixture.computed("value", fixture.value, Add{ .value = 1 }), + .b = fixture.computed("second", fixture.second, Add{ .value = 1 }), + .c = fixture.computed("value", fixture.value, Add{ .value = 2 }), + }; + try std.testing.expect(fixture.store.bindingOf(model.a.id).? == .compute); + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + // Nothing for the loader to fill itself, and one group. + try std.testing.expectEqual(0, loader.pending.len); + try std.testing.expectEqual(1, loader.queued.len); + try std.testing.expectEqual(2, loader.programs.items.len); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.a, &.{ 2, 3, 4, 5 }, .default); + try testing.expectShards(allocator, io, &buffers.b, &.{ 6, 7, 8, 9 }, .default); + try testing.expectShards(allocator, io, &buffers.c, &.{ 3, 4, 5, 6 }, .default); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); +} + +test "loader delivers host bindings into pinned host memory without charging the devices" { + for ([_]Memory{ .host_pinned, .host_mapped }) |kind| try expectHostDelivery(kind); +} + +fn expectHostDelivery(kind: Memory) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const platform = fixture.platform; + const pinned = fixture.store.view().createReplicatedTensor("second", null); + fixture.store.view().setMemory(pinned, kind); + try std.testing.expectEqual(kind, fixture.store.bindingOf(pinned.id).?.load.memory); + + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + const Model = struct { device: Tensor, host: Tensor }; + const model: Model = .{ .device = fixture.value, .host = pinned }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + + if (!backend.supportsMemory(platform, kind)) { + // Refused before anything is submitted. + try std.testing.expectError(error.InvalidBinding, loader.start(&model, &buffers, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + return; + } + try loader.start(&model, &buffers, &fixture.store, .none); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.device, &LoaderTestFixture.contents, .default); + // One shard per device, each in that device's pinned host memory space + // or, `host_mapped`, a view of pinned memory in its default space. + try testing.expectShards(allocator, io, &buffers.host, &LoaderTestFixture.second_contents, kind); + + // Only the tensor in device memory counts against the devices, on both + // sides of the admission ledger. + if (loader.backend.? == .direct) { + const direct = loader.backend.?.direct; + for (direct.sealed_bytes[0..platform.devices.len], direct.allocated_bytes[0..platform.devices.len]) |sealed, *allocated| { + try std.testing.expectEqual(LoaderTestFixture.contents.len, sealed); + try std.testing.expectEqual(sealed, allocated.load(.acquire)); + } + } +} + +test "a program frees its donated and non-donated inputs once" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); + const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); + defer allocator.free(path); + + const shape: Shape = .init(.{2}, .i32); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); + try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + + const Sum = struct { + donate: bool, + + pub fn forward(self: @This(), _: Shape, inputs: []const Tensor) Tensor { + const sum = inputs[0].add(inputs[1]); + return if (self.donate) sum.reuseBuffer(inputs[0]) else sum; + } + }; + for ([_]bool{ false, true }) |donate| { + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + const tensor = TensorStore.Source.bindProgram(&.{ + store.view().source("value", null).withPartitioning(.replicated, .{}), + store.view().source("weight", null).withPartitioning(.replicated, .{}), + }, shape, Sum{ .donate = donate }); + var output: Buffer = undefined; + try loader.start(&tensor, &output, &store, .none); + errdefer loader.cancel(); + try loader.programs.items[0].await(io, platform); + try std.testing.expectEqual(@as(usize, @intFromBool(donate)), loader.programs.items[0].exe.?.donated_input_indices.len); + // Retiring runs the program, then frees every input handle once, the + // donated one included. + try loader.wait(); + defer output.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + } +} + +test "cancel gives up on everything started without running its programs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Model = struct { value: Tensor, computed: Tensor }; + const model: Model = .{ .value = fixture.value, .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + loader.cancel(); + try LoaderTestFixture.expectFailed(&loader, error.Canceled); + try std.testing.expectEqual(@as(usize, 0), buffers.computed._shards.len); + // The loader is failed: what it waits for and what it is given. + try std.testing.expectError(error.Canceled, loader.wait()); + try std.testing.expectError(error.Canceled, loader.start(&model, &buffers, &fixture.store, .none)); + try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); +} + +test "cancel frees a published group's inputs without running its program" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Model = struct { computed: Tensor }; + const model: Model = .{ .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + // Publish the group as `wait` does, then give up before it retires. + try loader.publish(loader.queued.popFront().?); + try std.testing.expect(loader.programGroupPending()); + loader.cancel(); + try LoaderTestFixture.expectFailed(&loader, error.Canceled); + try std.testing.expect(!loader.programGroupPending()); + try std.testing.expectEqual(0, buffers.computed._shards.len); +} + +test "a read failure fails the loader for every later call" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // The broken group first, retired when the next one is admitted. + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); + defer loader.deinit(); + try loader.awaitSetup(); + loader.measured = null; + + const Model = struct { broken: Tensor, value: Tensor }; + const model: Model = .{ + .broken = fixture.computed("missing", fixture.missing, LoaderTestFixture.Identity{}), + .value = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + try std.testing.expectEqual(@as(usize, 0), buffers.value._shards.len); + try std.testing.expectError(error.FileNotFound, loader.start(&model, &buffers, &fixture.store, .none)); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); +} + +test "after a failed load every buffer is empty or whole, and deinitBufferized frees the tree" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + // A tree with a slice and optionals, some of it in the file that loads, + // a tensor in the file that does not exist, and a program over it. + const Layer = struct { weight: Tensor, bias: ?Tensor }; + const Model = struct { layers: []Layer, broken: Tensor, computed: Tensor }; + var layers = [_]Layer{ + .{ .weight = fixture.value, .bias = fixture.second }, + .{ .weight = fixture.second, .bias = null }, + }; + const model: Model = .{ + .layers = &layers, + .broken = fixture.missing, + .computed = fixture.computed("missing", fixture.missing, LoaderTestFixture.Identity{}), + }; + var buffers = try mem.bufferize(allocator, &model); + // Frees whatever the failed load wrote, under the testing allocator. + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + // Each buffer holds no shard or one per device. + const devices = fixture.platform.placedShape(fixture.value.shape())._sharding.mesh.?.devicesInCanonicalOrder().len; + var torn: usize = 0; + meta.forEachVisit(&buffers, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, shards: usize, count: *usize) void { + if (buffer._shards.len != 0 and buffer._shards.len != shards) count.* += 1; + } + }.call, .{ devices, &torn }); + try std.testing.expectEqual(0, torn); + try std.testing.expectEqual(0, buffers.computed._shards.len); +} + +test "the direct backend allocates what it was submitted once everything landed" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var placed: DeviceBytes = @splat(0); + // An empty tensor takes no bytes. + addPlacementBytes(&placed, fixture.platform, .init(.{ 0, 4 }, .f32)); + addPlacementBytes(&placed, fixture.platform, fixture.value.shape()); + for (placed[0..fixture.platform.devices.len]) |bytes| try std.testing.expectEqual(LoaderTestFixture.contents.len, bytes); + + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + const Model = struct { value: Tensor, computed: Tensor }; + const model: Model = .{ .value = fixture.value, .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + try loader.wait(); + if (loader.backend.? != .direct) return; + const direct = loader.backend.?.direct; + const n = fixture.platform.devices.len; + const unallocated = direct.unallocatedBytes(); + // Replicated program inputs and bulk outputs occupy every device. + for (direct.sealed_bytes[0..n], direct.allocated_bytes[0..n], unallocated[0..n]) |sealed, *allocated, bytes| { + try std.testing.expectEqual(LoaderTestFixture.contents.len * 2, sealed); + try std.testing.expectEqual(sealed, allocated.load(.acquire)); + try std.testing.expectEqual(0, bytes); + } +} + +test "init checks every option and returns InvalidOptions" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + const Case = struct { io: std.Io, opts: Loader.Options }; + const cases = [_]Case{ + .{ .io = io, .opts = .{ .read_parallelism = 0 } }, + .{ .io = io, .opts = .{ .read_parallelism = Loader.max_read_parallelism + 1 } }, + .{ .io = io, .opts = .{ .dma_block_size = 0 } }, + .{ .io = io, .opts = .{ .dma_block_size = backend.max_read_request_size + 1 } }, + // No backend of the VFS serves the scheme. + .{ .io = vfs.io(), .opts = .{ .checkpoint = "missing://bucket/model" } }, + }; + for (cases) |case| { + if (Loader.init(allocator, case.io, platform, case.opts)) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidOptions, err); + } + // Under a VFS, a local file may be read directly: a block is a whole + // number of alignment units, reads landing in blocks. + if (VFS.direct_io_alignment != 0) { + if (Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = "/model.safetensors", .dma_block_size = (8 << 20) + VFS.direct_io_alignment / 2 })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidOptions, err); + // Whatever the checkpoint: a request of one alignment unit cannot + // hold a direct read widened at each end. + try vfs.registerBackend("tiny", .{ .io = io, .read_hints = .{ .read_chunk_size = VFS.direct_io_alignment } }); + if (Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = "tiny://model", .dma_block_size = VFS.direct_io_alignment })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidOptions, err); + } + // A plain io has no VFS to ask: local sizing. Nothing started, so + // `deinit` cancels the set-up or destroys what it built. + var loader = try Loader.init(allocator, io, platform, .{ .checkpoint = "missing://bucket/model" }); + loader.deinit(); +} + +test "init copies a local checkpoint's path for its set-up" { + if (VFS.direct_io_alignment == 0) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const unit: [VFS.direct_io_alignment]u8 = @splat(7); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors", .data = &unit }); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; + var path_buffer: [std.fs.max_path_bytes + 32]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, "{s}/model.safetensors", .{root}); + + // The copy is the only allocation of `init`: nothing is left to free + // when it fails. + var failing: std.testing.FailingAllocator = .init(allocator, .{ .fail_index = 0 }); + if (Loader.init(failing.allocator(), vfs.io(), platform, .{ .checkpoint = path })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.OutOfMemory, err); + + // The set-up frees `init`'s copy once, whether it ran to its end or + // `deinit` canceled it before any `start`. + var loader = try Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = path }); + defer loader.deinit(); + try loader.awaitSetup(); + for ([_]VFS.DirectIo{ .on, .auto }) |policy| { + var unstarted = try Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = path, .direct_io = policy }); + unstarted.deinit(); + } +} + +test "a failed set-up fails the first start, and deinit frees the rest" { + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(std.testing.allocator, io); + defer fixture.deinit(); + // `init` allocates nothing; the set-up's third allocation, inside the + // backend, fails, so the backend unwinds what it built before it. + var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 2 }); + var loader = try Loader.init(failing.allocator(), io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var output: Buffer = undefined; + try std.testing.expectError(error.OutOfMemory, loader.start(&fixture.value, &output, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.OutOfMemory); + try std.testing.expect(loader.backend == null); + try std.testing.expectError(error.OutOfMemory, loader.wait()); +} + +test "without concurrency init sets up in place" { + const allocator = std.testing.allocator; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, std.testing.io); + defer fixture.deinit(); + // The testing io, but `io.concurrent` refuses and `io.async` runs the + // function in place. + var vtable = std.testing.io.vtable.*; + vtable.concurrent = struct { + fn concurrent(_: ?*anyopaque, _: usize, _: std.mem.Alignment, _: []const u8, _: std.mem.Alignment, _: *const fn (*const anyopaque, *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { + return error.ConcurrencyUnavailable; + } + }.concurrent; + vtable.async = struct { + fn async(_: ?*anyopaque, result: []u8, _: std.mem.Alignment, context: []const u8, _: std.mem.Alignment, start: *const fn (*const anyopaque, *anyopaque) void) ?*std.Io.AnyFuture { + start(context.ptr, result.ptr); + return null; + } + }.async; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try std.testing.expect(loader.setup.?.any_future == null); + // Nothing started: neither has anything to do. + loader.cancel(); + try loader.wait(); + try std.testing.expectEqual(null, loader.failure); + + var output: Buffer = undefined; + try loader.start(&fixture.value, &output, &fixture.store, .none); + try loader.wait(); + defer output.deinit(); + try testing.expectShards(allocator, io, &output, &LoaderTestFixture.contents, .default); +} + +test "a rate-limited HTTP source loads through the VFS hold" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const tensor_bytes = 1024; + const tensors = 4; + + // Four tensor ranges with a gap between them, so the planner keeps four + // jobs and the load makes four concurrent GETs. + var object: [(2 * tensors - 1) * tensor_bytes]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 31 +% 7); + + var server = try VFS.MockServer.init(io, &object, .{ + // Two GETs per 100 ms: the four readers are throttled repeatedly. + .throttle = .{ .window = .{ .gets = 2, .per_ms = 100 } }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var http: VFS.HTTP = try .init(allocator, io, &client, .{ .protocol = .http, .retry = .{ + .max_retries = 5, + .initial_delay = .fromMilliseconds(20), + .max_delay = .fromMilliseconds(20), + .max_hold = .fromSeconds(5), + .throttle_budget = .fromSeconds(30), + } }); + defer http.deinit(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + try vfs.registerBackend("http", http.backend()); + const stats = &http.governor.stats; + + var url_buffer: [128]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + + const platform = testing.env(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + const names = [tensors][]const u8{ "a", "b", "c", "d" }; + for (names, 0..) |name, index| { + try registry.registerTensor(.{ + .file_uri = url, + .name = name, + .shape = .init(.{tensor_bytes}, .u8), + .offset = 2 * index * tensor_bytes, + }); + } + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + + const Model = struct { a: Tensor, b: Tensor, c: Tensor, d: Tensor }; + const model: Model = .{ + .a = store.view().createReplicatedTensor("a", null), + .b = store.view().createReplicatedTensor("b", null), + .c = store.view().createReplicatedTensor("c", null), + .d = store.view().createReplicatedTensor("d", null), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + + var loader = try Loader.init(allocator, vfs.io(), platform, .{ + .checkpoint = url, + .read_parallelism = 4, + }); + // Holding costs time, not credits: the direct backend's `destroy` + // asserts that every lifecycle credit came back with its pinned blocks. + // It checks once its workers stopped, since until then one may hold a + // credit for a claim that finds nothing. + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + inline for (comptime std.meta.fieldNames(Model), 0..) |field_name, index| { + try testing.expectShards(allocator, io, &@field(buffers, field_name), object[2 * index * tensor_bytes ..][0..tensor_bytes], .default); + } + + try server.stop(io); + + // The server rate limited the load, the backend held every reader, and + // no reader burned its retry budget for it. + try std.testing.expect(stats.throttles.load(.acquire) > 0); + try std.testing.expect(stats.holds.load(.acquire) > 0); + try std.testing.expect(stats.hold_wait_ns.load(.acquire) > 0); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, tensors * tensor_bytes), stats.physical_bytes.load(.acquire)); + + // The checkpoint's VFS backend, not the local hints (8 MiB), sized the + // reads: 16 MiB requests over the CPU's 2 MiB blocks. Its file was read + // exact: a remote file is never direct. + const direct = loader.backend.?.direct; + try std.testing.expectEqual(@as(usize, 16 << 20), direct.plan_config.request_size); + try std.testing.expectEqual(@as(usize, 0), direct.source_slots.get(url).?.alignment orelse 0); +} + +test "a rejected binding or a program of the wrong output fails the loader" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const platform = fixture.platform; + // Two rows of two, not the four bytes the tensor holds. + const Rows = struct { + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0].reshape(.{ 2, 2 }); + } + }; + // A mesh of devices the platform does not have: the same tensors placed + // on it, in device and host memory, and a program's output. + var foreign_physical = platform.replicated_mesh.physical.*; + const foreign_devices = try allocator.dupe(Sharding.Device, foreign_physical.devices_in_canonical_order); + defer allocator.free(foreign_devices); + for (foreign_devices) |*device| device.id += 100; + foreign_physical.devices_in_canonical_order = foreign_devices; + var foreign_mesh = platform.replicated_mesh.*; + foreign_mesh.physical = &foreign_physical; + const pinned = fixture.store.view().createReplicatedTensor("second", null); + fixture.store.view().setMemory(pinned, .host_pinned); + var foreign = [_]Tensor{ fixture.value, pinned, fixture.value }; + for (&foreign) |*tensor| tensor._shape._sharding.mesh = &foreign_mesh; + // The program places its output replicated, the tensor expects its + // bytes split over the devices: same dims, another placement. It needs + // devices that split four bytes. + var misplaced = fixture.value; + misplaced._shape = Shape.init(.{ .x = 4 }, .u8).withPartitioning(platform.meshes.get("model").?, .{ .x = .model }); + const splits = platform.devices.len > 1 and 4 % platform.devices.len == 0; + + // No binding, an empty source or program input, a mesh of other + // devices, and a program of the wrong shape or placement. + const Case = struct { tensor: Tensor, at_wait: bool = false, skip: bool = false }; + const cases = [_]Case{ + .{ .tensor = Tensor.fromShape(fixture.value.shape()) }, + .{ .tensor = fixture.empty }, + .{ .tensor = fixture.computed("empty", fixture.empty, LoaderTestFixture.Identity{}) }, + .{ .tensor = foreign[0] }, + .{ .tensor = foreign[1] }, + .{ .tensor = fixture.computed("value", foreign[2], LoaderTestFixture.Identity{}) }, + .{ .tensor = fixture.computed("value", fixture.value, Rows{}), .at_wait = true }, + .{ .tensor = fixture.computed("value", misplaced, LoaderTestFixture.Identity{}), .at_wait = true, .skip = !splits }, + }; + for (cases) |case| { + if (case.skip) continue; + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var output: Buffer = undefined; + if (case.at_wait) { + try loader.start(&case.tensor, &output, &fixture.store, .none); + try std.testing.expectError(error.InvalidBinding, loader.wait()); + } else { + try std.testing.expectError(error.InvalidBinding, loader.start(&case.tensor, &output, &fixture.store, .none)); + } + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + // `start` emptied the buffer before it failed. + try std.testing.expectEqual(0, output._shards.len); + output.deinit(); + try std.testing.expectError(error.InvalidBinding, loader.start(&fixture.value, &output, &fixture.store, .none)); + } +} + +test "buffers of another structure fail start before any buffer is written" { + var fixture: LoaderTestFixture = undefined; + try fixture.init(std.testing.allocator, std.testing.io); + defer fixture.deinit(); + const value = fixture.value; + const second = fixture.second; + + // Each pair holds as many tensors on both sides, in another structure, + // and a pair precedes each mismatch. + const Optionals = struct { head: Tensor, a: ?Tensor, b: ?Tensor }; + try expectStructureMismatch(&fixture, Optionals, .{ .head = value, .a = value, .b = null }, .{ .head = value, .a = null, .b = value }); + const Arms = struct { head: Tensor, arm: union(enum) { a: Tensor, b: Tensor }, last: Tensor }; + try expectStructureMismatch(&fixture, Arms, .{ .head = value, .arm = .{ .a = value }, .last = second }, .{ .head = value, .arm = .{ .b = value }, .last = second }); + const Slices = struct { head: Tensor, first: []Tensor, second: []Tensor }; + var two = [_]Tensor{ value, second }; + var one = [_]Tensor{value}; + var other_one = [_]Tensor{second}; + var other_two = [_]Tensor{ second, value }; + try expectStructureMismatch(&fixture, Slices, .{ .head = value, .first = &two, .second = &one }, .{ .head = value, .first = &other_one, .second = &other_two }); +} + +/// Starts `model` into the buffers bufferized from `built`, which has as +/// many tensors in another structure: `InvalidBinding`, and every buffer as +/// it was. +fn expectStructureMismatch(fixture: *LoaderTestFixture, comptime Model: type, built: Model, model: Model) !void { + const allocator = std.testing.allocator; + var loader = try Loader.init(allocator, std.testing.io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var buffers = try mem.bufferize(allocator, &built); + defer mem.deinitBufferized(allocator, &buffers); + // No tensor has this shape, which `start` would replace with its + // tensor's when it empties a buffer. + const untouched: Shape = .init(.{ 1, 2, 3 }, .u8); + meta.forEachVisit(&buffers, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, shape: Shape) void { + buffer._shape = shape; + } + }.call, .{untouched}); + try std.testing.expectError(error.InvalidBinding, loader.start(&model, &buffers, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + var written: usize = 0; + meta.forEachVisit(&buffers, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, shape: Shape, count: *usize) void { + if (!buffer._shape.eql(shape) or buffer._shards.len != 0) count.* += 1; + } + }.call, .{ untouched, &written }); + try std.testing.expectEqual(0, written); +} + +/// Two safetensors files for composite loads, every row 16 bytes (8 u16): +/// `first` holds `a` [3, 8] then `b` [2, 8], adjacent, then `spacer`, then +/// `q` [8, 8], `k` [4, 8] and `v` [4, 8]; `second` holds `plain` [4, 8], +/// then `c` [4, 8], the experts `e{0..3}.w1` and `e{0..3}.w3` [2, 8] and +/// `wide` [64, 8]. `empty` [0, 8] is registered without bytes. Pinned after +/// `init`. Each store takes a copy of `registry` (`testing.storeOf`). +const CompositeFixture = struct { + tmp: std.testing.TmpDir, + registry: safetensors.TensorRegistry, + a: [48]u8, + b: [32]u8, + c: [64]u8, + plain: [64]u8, + q: [128]u8, + k: [64]u8, + v: [64]u8, + w1: [4][32]u8, + w3: [4][32]u8, + wide: [1024]u8, + + const expert_names = [2][4][]const u8{ + .{ "e0.w1", "e1.w1", "e2.w1", "e3.w1" }, + .{ "e0.w3", "e1.w3", "e2.w3", "e3.w3" }, + }; + + fn init(self: *CompositeFixture, allocator: std.mem.Allocator, io: std.Io) !void { + self.tmp = std.testing.tmpDir(.{}); + errdefer self.tmp.cleanup(); + self.registry = .init(allocator); + errdefer self.registry.deinit(); + testing.pattern(&self.a, 1); + testing.pattern(&self.b, 2); + testing.pattern(&self.c, 3); + testing.pattern(&self.plain, 4); + testing.pattern(&self.q, 9); + testing.pattern(&self.k, 10); + testing.pattern(&self.v, 11); + for (&self.w1, &self.w3, 0..) |*w1, *w3, e| { + testing.pattern(w1, @intCast(20 + e)); + testing.pattern(w3, @intCast(30 + e)); + } + var spacer: [16]u8 = undefined; + testing.pattern(&spacer, 5); + testing.pattern(&self.wide, 6); + try testing.writeSafetensors(allocator, io, self.tmp.dir, "first.safetensors", &self.registry, &.{ + .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &self.a }, + .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.b }, + .{ .name = "spacer", .dtype = "U16", .dims = &.{ 1, 8 }, .bytes = &spacer }, + .{ .name = "q", .dtype = "U16", .dims = &.{ 8, 8 }, .bytes = &self.q }, + .{ .name = "k", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.k }, + .{ .name = "v", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.v }, + }); + try testing.writeSafetensors(allocator, io, self.tmp.dir, "second.safetensors", &self.registry, &.{ + .{ .name = "plain", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.plain }, + .{ .name = "c", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.c }, + .{ .name = expert_names[0][0], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[0] }, + .{ .name = expert_names[1][0], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[0] }, + .{ .name = expert_names[0][1], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[1] }, + .{ .name = expert_names[1][1], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[1] }, + .{ .name = expert_names[0][2], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[2] }, + .{ .name = expert_names[1][2], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[2] }, + .{ .name = expert_names[0][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[3] }, + .{ .name = expert_names[1][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[3] }, + .{ .name = "wide", .dtype = "U16", .dims = &.{ 64, 8 }, .bytes = &self.wide }, + }); + try self.registry.registerTensor(.{ + .file_uri = self.registry.tensors.get("a").?.file_uri, + .name = "empty", + .shape = .init(.{ 0, 8 }, .u16), + .offset = 0, + }); + } + + fn deinit(self: *CompositeFixture) void { + self.registry.deinit(); + self.tmp.cleanup(); + } + + fn rows(view: TensorStore.View, name: []const u8) TensorStore.Source { + return view.source(name, .{ .row, .col }); + } +}; + +test "loader assembles concatenated and padded sources on every placement and memory kind" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // a, b and c end to end over 16 rows; c alone over 8 rows; b then a + // over 6 rows. The rest is zero. + var concatenated: [256]u8 = @splat(0); + @memcpy(concatenated[0..48], &fixture.a); + @memcpy(concatenated[48..80], &fixture.b); + @memcpy(concatenated[80..144], &fixture.c); + var padded: [128]u8 = @splat(0); + @memcpy(padded[0..64], &fixture.c); + var reversed: [96]u8 = @splat(0); + @memcpy(reversed[0..32], &fixture.b); + @memcpy(reversed[32..80], &fixture.a); + + const Source = TensorStore.Source; + const rows = CompositeFixture.rows; + const Model = struct { concatenated: Tensor, padded: Tensor, reversed: Tensor, plain: Tensor }; + for ([_]usize{ 2, 4 }) |device_count| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const platform = devices.platform; + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + // Rows over the devices (b and c straddle shards on four), every + // row split over them, and every byte on each of them. + inline for (.{ .{ .row = .model }, .{ .col = .model }, .{} }) |partitioning| { + const shards: u64 = if (@typeInfo(@TypeOf(partitioning)).@"struct".field_names.len == 0) 1 else device_count; + for ([_]Memory{ .default, .host_pinned, .host_mapped }) |memory| { + if (!backend.supportsMemory(platform, memory)) continue; + const view = store.view(); + const model: Model = .{ + .concatenated = Source.concatenate(&.{ rows(view, "a"), rows(view, "b"), rows(view, "c") }, .row) + .pad(0, .{ .row = Source.Pad{ .high = 7 } }) + .withPartitioning(.tp, partitioning).bind(), + // The padded engram-style table: one source. + .padded = rows(view, "c").pad(0, .{ .row = Source.Pad{ .high = 4 } }).withPartitioning(.tp, partitioning).bind(), + // Out of file order, on the platform's replicated mesh. + .reversed = Source.concatenate(&.{ rows(view, "b"), rows(view, "a") }, .row) + .pad(0, .{ .row = Source.Pad{ .high = 1 } }) + .withPartitioning(.replicated, .{}).bind(), + .plain = rows(view, "plain").withPartitioning(.tp, partitioning).bind(), + }; + inline for (comptime std.meta.fieldNames(Model)) |name| view.setMemory(@field(model, name), memory); + try std.testing.expect(store.bindingOf(model.plain.id).?.load.source.whole() != null); + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + try testing.expectShards(allocator, io, &buffers.concatenated, &concatenated, memory); + try testing.expectShards(allocator, io, &buffers.padded, &padded, memory); + try testing.expectShards(allocator, io, &buffers.reversed, &reversed, memory); + try testing.expectShards(allocator, io, &buffers.plain, &fixture.plain, memory); + // Each destination once, padding included. + try std.testing.expectEqual(@as(usize, 256 + 128 + 96 + 64), loader.stats.bytes_loaded); + // A destination is allocated once per device, whatever its + // parts; host memory counts on neither side. + if (loader.backend.? == .direct) { + const direct = loader.backend.?.direct; + const n = platform.devices.len; + for (direct.sealed_bytes[0..n], direct.allocated_bytes[0..n]) |sealed, *allocated| try std.testing.expectEqual(sealed, allocated.load(.acquire)); + for (devices.physical.devices_in_canonical_order) |device| { + const expected: u64 = if (memory.isHost()) 0 else (256 + 128 + 64) / shards + 96; + try std.testing.expectEqual(expected, direct.sealed_bytes[device.id]); + } + } + } + } + } +} + +test "loader assembles interleaved, stacked and split sources" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const Source = TensorStore.Source; + const rows = CompositeFixture.rows; + const row = 16; + + // Experts with their two projections concatenated, then row by row. + var concatenated_experts: [256]u8 = undefined; + var interleaved_experts: [256]u8 = undefined; + for (0..4) |e| { + @memcpy(concatenated_experts[e * 64 ..][0..32], &fixture.w1[e]); + @memcpy(concatenated_experts[e * 64 + 32 ..][0..32], &fixture.w3[e]); + for (0..2) |r| { + @memcpy(interleaved_experts[e * 64 + r * 2 * row ..][0..row], fixture.w1[e][r * row ..][0..row]); + @memcpy(interleaved_experts[e * 64 + (r * 2 + 1) * row ..][0..row], fixture.w3[e][r * row ..][0..row]); + } + } + + const Model = struct { qkv: Tensor, concatenated: Tensor, interleaved: Tensor, head: Tensor, tail: Tensor }; + for ([_]usize{ 2, 4 }) |device_count| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const platform = devices.platform; + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + const view = store.view(); + const groups: i64 = @intCast(device_count); + + // q, k and v cut in one group per device: each device holds its rows + // of every member, contiguous. Every part has `groups` runs. + var members: [3]Source = undefined; + for (&members, [_][]const u8{ "q", "k", "v" }) |*member, name| { + member.* = rows(view, name).splitAxis(.row, .{ .group = groups, .r = -1 }); + } + var experts: [2]Source = undefined; + for (&experts, CompositeFixture.expert_names) |*stacked, names| { + var per_expert: [4]Source = undefined; + for (&per_expert, names) |*expert, name| expert.* = rows(view, name); + stacked.* = Source.stack(&per_expert, 0, .expert); + } + const fused = Source.concatenate(&.{ rows(view, "q"), rows(view, "k") }, .row); + // A part of q, and the rest of q with all of k. + const head, const tail = fused.split(.row, &.{ 4, 8 })[0..2].*; + const model: Model = .{ + .qkv = Source.concatenate(&members, .r).merge(.{ .row = .{ .group, .r } }).withPartitioning(.tp, .{ .row = .model }).bind(), + // Experts are whole rows of the merged axis, one or two per device. + .concatenated = Source.concatenate(&experts, .row) + .merge(.{ .rows = .{ .expert, .row } }) + .withPartitioning(.tp, .{ .rows = .model }).bind(), + .interleaved = Source.stack(&experts, 2, .proj) + .merge(.{ .rows = .{ .expert, .row, .proj } }) + .withPartitioning(.tp, .{ .rows = .model }).bind(), + .head = head.withPartitioning(.replicated, .{}).bind(), + .tail = tail.withPartitioning(.tp, .{ .row = .model }).bind(), + }; + var lowering: std.heap.ArenaAllocator = .init(allocator); + defer lowering.deinit(); + const qkv_parts = try store.bindingOf(model.qkv.id).?.load.source.lower(lowering.allocator()); + try std.testing.expectEqual(@as(usize, 3), qkv_parts.len); + for (qkv_parts) |part| try std.testing.expectEqual(@as(u64, device_count), part.lattice.runs()); + + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + var qkv: [256]u8 = undefined; + var at: usize = 0; + for (0..device_count) |g| { + for ([_][]const u8{ &fixture.q, &fixture.k, &fixture.v }) |member| { + const per_group = member.len / device_count; + @memcpy(qkv[at..][0..per_group], member[g * per_group ..][0..per_group]); + at += per_group; + } + } + try testing.expectShards(allocator, io, &buffers.qkv, &qkv, .default); + try testing.expectShards(allocator, io, &buffers.concatenated, &concatenated_experts, .default); + try testing.expectShards(allocator, io, &buffers.interleaved, &interleaved_experts, .default); + try testing.expectShards(allocator, io, &buffers.head, fixture.q[0..64], .default); + var tail_bytes: [128]u8 = undefined; + @memcpy(tail_bytes[0..64], fixture.q[64..128]); + @memcpy(tail_bytes[64..], &fixture.k); + try testing.expectShards(allocator, io, &buffers.tail, &tail_bytes, .default); + } +} + +test "loader gathers strided sources: a row interleave across files, broadcasts and a transpose" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const Source = TensorStore.Source; + // Rows of 16 bytes: g and u interleave row by row, 96 runs each; s0 and + // s1 repeat 100 and 60 times; z [24, 4] is transposed, 96 runs of two + // bytes, then padded with two zero rows. Every part has more runs than + // expand, so the direct backend gathers them, its jobs reading g's rows + // from one file and u's from the other. + var g: [96 * 16]u8 = undefined; + var u: [96 * 16]u8 = undefined; + var z: [24 * 8]u8 = undefined; + var s0: [4]u8 = undefined; + var s1: [4]u8 = undefined; + testing.pattern(&g, 1); + testing.pattern(&u, 2); + testing.pattern(&z, 3); + testing.pattern(&s0, 4); + testing.pattern(&s1, 5); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "first.safetensors", ®istry, &.{ + .{ .name = "g", .dtype = "U16", .dims = &.{ 96, 8 }, .bytes = &g }, + .{ .name = "s0", .dtype = "F32", .dims = &.{}, .bytes = &s0 }, + .{ .name = "z", .dtype = "U16", .dims = &.{ 24, 4 }, .bytes = &z }, + }); + try testing.writeSafetensors(allocator, io, tmp.dir, "second.safetensors", ®istry, &.{ + .{ .name = "s1", .dtype = "F32", .dims = &.{}, .bytes = &s1 }, + .{ .name = "u", .dtype = "U16", .dims = &.{ 96, 8 }, .bytes = &u }, + }); + + var interleaved: [192 * 16]u8 = undefined; + for (0..96) |r| { + @memcpy(interleaved[2 * r * 16 ..][0..16], g[r * 16 ..][0..16]); + @memcpy(interleaved[(2 * r + 1) * 16 ..][0..16], u[r * 16 ..][0..16]); + } + var scales: [160 * 4]u8 = undefined; + for (0..160) |i| @memcpy(scales[i * 4 ..][0..4], if (i < 100) &s0 else &s1); + var transposed: [6 * 48]u8 = @splat(0); + for (0..4) |column| { + for (0..24) |row| @memcpy(transposed[column * 48 + row * 2 ..][0..2], z[row * 8 + column * 2 ..][0..2]); + } + + const Model = struct { by_rows: Tensor, by_columns: Tensor, scales_by_rows: Tensor, scales_by_columns: Tensor, transposed: Tensor, transposed_by_columns: Tensor }; + for ([_]usize{ 1, 2, 4 }) |device_count| { + for ([_]usize{ 0, 64 }) |alignment| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, ®istry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + const view = store.view(); + const rows = Source.stack(&.{ view.source("g", .{ .row, .col }), view.source("u", .{ .row, .col }) }, 1, .proj) + .merge(.{ .row = .{ .row, .proj } }); + const repeated = Source.concatenate(&.{ + view.source("s0", .{}).broad(Shape.init(.{ .c = 100 }, .f32)), + view.source("s1", .{}).broad(Shape.init(.{ .c = 60 }, .f32)), + }, .c).splitAxis(.c, .{ .row = 40, .col = 4 }); + const flipped = view.source("z", .{ .row, .col }).transpose(.{ .col, .row }).withTags(.{ .row, .col }) + .pad(0, .{ .row = Tensor.Pad{ .high = 2 } }); + const model: Model = .{ + .by_rows = rows.withPartitioning(.tp, .{ .row = .model }).bind(), + .by_columns = rows.withPartitioning(.tp, .{ .col = .model }).bind(), + .scales_by_rows = repeated.withPartitioning(.tp, .{ .row = .model }).bind(), + .scales_by_columns = repeated.withPartitioning(.tp, .{ .col = .model }).bind(), + .transposed = flipped.withPartitioning(.replicated, .{}).bind(), + .transposed_by_columns = flipped.withPartitioning(.tp, .{ .col = .model }).bind(), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, devices.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + // Jobs of 128 bytes: every destination takes several. + try loader.awaitSetup(); + switch (loader.backend.?) { + .direct => |direct| { + direct.plan_config.request_size = 256; + direct.reads.alignment = alignment; + }, + .buffered => {}, + } + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.by_rows, &interleaved, .default); + try testing.expectShards(allocator, io, &buffers.by_columns, &interleaved, .default); + try testing.expectShards(allocator, io, &buffers.scales_by_rows, &scales, .default); + try testing.expectShards(allocator, io, &buffers.scales_by_columns, &scales, .default); + try testing.expectShards(allocator, io, &buffers.transposed, &transposed, .default); + try testing.expectShards(allocator, io, &buffers.transposed_by_columns, &transposed, .default); + } + } +} + +test "loader gathers a block-scale swizzle, its padding zero" { + const allocator = std.testing.allocator; + const io = std.testing.io; + // Scales [e=4, m=31, k=6] in the 128x4 layout at 8x4: m padded to 32 + // and tiled by 2x4, k padded to 8. Its parts repeat runs of four bytes + // or two, more than expand, so the direct backend gathers them. + var scales: [4 * 31 * 6]u8 = undefined; + testing.pattern(&scales, 7); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "scales.safetensors", ®istry, &.{ + .{ .name = "scales", .dtype = "U8", .dims = &.{ 4, 31, 6 }, .bytes = &scales }, + }); + var swizzled: [4 * 32 * 8]u8 = @splat(0); + for (0..4) |e| for (0..4) |mo| for (0..2) |ko| for (0..4) |mi| for (0..2) |m4| for (0..4) |ki| { + const m = mo * 8 + m4 * 4 + mi; + const k = ko * 4 + ki; + const flat = (((mo * 2 + ko) * 4 + mi) * 2 + m4) * 4 + ki; + if (m < 31 and k < 6) swizzled[e * 256 + flat] = scales[(e * 31 + m) * 6 + k]; + }; + + const Model = struct { replicated: Tensor, by_rows: Tensor }; + for ([_]usize{ 1, 2, 4 }) |device_count| { + for ([_]usize{ 0, 64 }) |alignment| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, ®istry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + const swizzle = store.view().source("scales", .{ .e, .m, .k }) + .pad(0, .{ .m = Tensor.Pad{ .high = 1 }, .k = Tensor.Pad{ .high = 2 } }) + .splitAxis(.m, .{ .mo = -1, .m4 = 2, .mi = 4 }) + .splitAxis(.k, .{ .ko = -1, .ki = 4 }) + .transpose(.{ .e, .mo, .ko, .mi, .m4, .ki }) + .merge(.{ .flat = .{ .mo, .ko, .mi, .m4, .ki } }); + const model: Model = .{ + .replicated = swizzle.withPartitioning(.replicated, .{}).bind(), + .by_rows = swizzle.withPartitioning(.tp, .{ .e = .model }).bind(), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, devices.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + // Jobs of at most 1024 bytes, each reading the 744 source + // bytes whole: the swizzle reorders its runs. + try loader.awaitSetup(); + switch (loader.backend.?) { + .direct => |direct| { + direct.plan_config.request_size = 1024; + direct.reads.alignment = alignment; + }, + .buffered => {}, + } + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.replicated, &swizzled, .default); + try testing.expectShards(allocator, io, &buffers.by_rows, &swizzled, .default); + } + } +} + +test "bindings the loader cannot fill fail it before submission" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // Three nibbles: whole bytes hold no element boundary past the first two. + try fixture.registry.registerTensor(.{ + .file_uri = fixture.registry.tensors.get("a").?.file_uri, + .name = "nibbles", + .shape = .init(.{3}, .u4), + .offset = 0, + }); + try fixture.registry.registerTensor(.{ + .file_uri = fixture.registry.tensors.get("a").?.file_uri, + .name = "nibble_rows", + .shape = .init(.{ 2, 3 }, .f4e2m1), + .offset = 0, + }); + const platform = testing.env(); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + const Source = TensorStore.Source; + const view = store.view(); + const rows = CompositeFixture.rows; + + const rejected = [_]Tensor{ + // Nothing to read: an empty tensor padded. + rows(view, "empty").pad(0, .{ .row = Source.Pad{ .high = 8 } }).withPartitioning(.replicated, .{}).bind(), + // Twelve bits: its one run ends inside a byte (no known checkpoint + // has one). + view.createReplicatedTensor("nibbles", null), + // Strided parts into host memory. + host: { + const tensor = rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{}).bind(); + view.setMemory(tensor, .host_pinned); + break :host tensor; + }, + // Three whole bytes, in rows of three nibbles that do not pack, as a + // tensor and as a program input. + view.createReplicatedTensor("nibble_rows", null), + Source.bindProgram(&.{view.source("nibble_rows", null).withPartitioning(.replicated, .{})}, .init(.{3}, .u8), LoaderTestFixture.Identity{}), + // A program input whose transpose reads its 1024 source bytes in + // every job, more than one read. + Source.bindProgram(&.{rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{})}, .init(.{ 8, 64 }, .u16), LoaderTestFixture.Identity{}), + }; + var output: Buffer = undefined; + for (&rejected) |*tensor| { + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + // One read holds half of `wide`; the buffered backend reads any part. + try loader.awaitSetup(); + switch (loader.backend.?) { + .direct => |direct| direct.plan_config.request_size = 512, + .buffered => if (tensor == &rejected[rejected.len - 1]) continue, + } + try std.testing.expectError(error.InvalidBinding, loader.start(tensor, &output, &store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + } +} + +test "loader fills gaps, reordered and strided rows, and a transpose" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const platform = testing.env(); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + const Source = TensorStore.Source; + const Pad = Source.Pad; + const view = store.view(); + const rows = CompositeFixture.rows; + const a = rows(view, "a"); + + // b's second row past a gap, the rest zero; b then a, each padded; a's + // last row before its first two, one source feeding two parts; a's + // rows every other row, a strided part of few runs, expanded; and a + // transpose, 512 runs of two bytes, gathered. + const Model = struct { sparse: Tensor, padded: Tensor, swapped: Tensor, strided: Tensor, transposed: Tensor }; + const model: Model = .{ + .sparse = rows(view, "b").slice(.row, .{ .start = 1 }).pad(0, .{ .row = Pad{ .low = 4, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), + .padded = Source.concatenate(&.{ + rows(view, "b").pad(0, .{ .row = Pad{ .high = 2 } }), + a.pad(0, .{ .row = Pad{ .high = 1 } }), + }, .row).withPartitioning(.replicated, .{}).bind(), + .swapped = Source.concatenate(&.{ a.slice(.row, .{ .start = 2 }), a.slice(.row, .{ .end = 2 }) }, .row) + .pad(0, .{ .row = Pad{ .high = 5 } }).withPartitioning(.replicated, .{}).bind(), + .strided = a.pad(0, .{ .row = Pad{ .interior = 1, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), + .transposed = rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{}).bind(), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + var expected: [128]u8 = @splat(0); + @memcpy(expected[64..80], fixture.b[16..32]); + try testing.expectShards(allocator, io, &buffers.sparse, &expected, .default); + expected = @splat(0); + @memcpy(expected[0..32], &fixture.b); + @memcpy(expected[64..112], &fixture.a); + try testing.expectShards(allocator, io, &buffers.padded, &expected, .default); + expected = @splat(0); + @memcpy(expected[0..16], fixture.a[32..48]); + @memcpy(expected[16..48], fixture.a[0..32]); + try testing.expectShards(allocator, io, &buffers.swapped, &expected, .default); + expected = @splat(0); + for (0..3) |i| @memcpy(expected[i * 32 ..][0..16], fixture.a[i * 16 ..][0..16]); + try testing.expectShards(allocator, io, &buffers.strided, &expected, .default); + var transposed_bytes: [1024]u8 = undefined; + for (0..8) |column| { + for (0..64) |row| @memcpy(transposed_bytes[column * 128 + row * 2 ..][0..2], fixture.wide[row * 16 + column * 2 ..][0..2]); + } + try testing.expectShards(allocator, io, &buffers.transposed, &transposed_bytes, .default); +} + +test "a program takes its inputs from sources, in order" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const Source = TensorStore.Source; + const view = fixture.store.view(); + + // value's first half then second's last half, assembled by the loader, + // and second as it is: the program subtracts the second input from the + // first, which tells them apart. + const Sub = struct { + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0].sub(inputs[1]); + } + }; + const assembled = Source.concatenate(&.{ + view.source("value", .{.x}).slice(.x, .{ .end = 2 }), + view.source("second", .{.x}).slice(.x, .{ .start = 2 }), + }, .x); + const second = view.source("second", .{.x}); + const inputs = [_]Source.Placed{ assembled.withPartitioning(.replicated, .{}), second.withPartitioning(.replicated, .{}) }; + const tensor = Source.bindProgram(&inputs, second.shape(), Sub{}); + const compute = fixture.store.bindingOf(tensor.id).?.compute; + try std.testing.expectEqual(2, compute.inputs.len); + try std.testing.expect(compute.program.as(Sub) != null and compute.program.as(LoaderTestFixture.Identity) == null); + try std.testing.expect(compute.inputs[0].whole() == null); + try std.testing.expect(compute.inputs[1].whole() != null); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var output: Buffer = undefined; + try loader.start(&tensor, &output, &fixture.store, .none); + try loader.wait(); + defer output.deinit(); + // { 1, 2, 7, 8 } - { 5, 6, 7, 8 }, wrapping. + try testing.expectShards(allocator, io, &output, &.{ 252, 252, 0, 0 }, .default); +} + +test "a padded host-mapped table stays in host memory and serves device gathers in place" { + // The engram arrangement: one source of 264-byte rows, an odd row count + // padded to a multiple of 8, rows sharded over every device, delivered + // as device-kind views of pinned host memory. A program gathers rows + // from each device's local shard. + try expectHostTableGathers(.rows); +} + +test "a replicated host-mapped table is one allocation that every device reads in place" { + // The same table on every device: the direct loader makes it one + // allocation (`direct_loader.DestinationState.initHost`), each device's + // shard a view of it, and every device's kernels gather from it. + try expectHostTableGathers(.replicated); +} + +/// Loads a padded table as `host_mapped`, rows sharded over every device or +/// replicated on each, and gathers rows of each device's shard on that +/// device. The table is freed while the gather is queued behind matrix +/// products: its memory must outlive the read, and go once it is done. +fn expectHostTableGathers(placement: enum { rows, replicated }) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const rows: usize = 100_003; + const row_bytes: usize = 264; + const gathered: usize = 256; + + // The accelerator under test, or four CPU devices. + const shared = testing.env(); + const owned: ?*Platform = if (shared.target == .cpu) + try Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 4 } }) + else + null; + defer if (owned) |platform| platform.deinit(allocator, io); + const platform: *const Platform = if (owned) |p| p else shared; + const api = platform.pjrt_api; + const mesh: *const Sharding.Mesh = if (owned) |p| + try p.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) + else + shared.meshes.get("model").?; + if (!backend.supportsMemory(platform, .host_mapped)) return error.SkipZigTest; + const devices = mesh.devicesInCanonicalOrder(); + const shard_count = devices.len; + + const padded_rows = std.mem.alignForward(usize, rows, 8); + const shard_rows = if (placement == .rows) padded_rows / shard_count else padded_rows; + if (placement == .rows) try std.testing.expectEqual(padded_rows, shard_rows * shard_count); + const padded = try allocator.alloc(u8, padded_rows * row_bytes); + defer allocator.free(padded); + @memset(padded, 0); + for (0..rows) |row| { + const bytes = padded[row * row_bytes ..][0..row_bytes]; + // The row's number first, so a misplaced row cannot pass for another. + std.mem.writeInt(u32, bytes[0..4], @intCast(row + 1), .little); + for (bytes[4..], 4..) |*byte, column| byte.* = @as(u8, @truncate(row *% 131 +% column *% 7)) | 1; + } + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "table.safetensors", ®istry, &.{ + .{ .name = "table", .dtype = "U8", .dims = &.{ @intCast(rows), @intCast(row_bytes) }, .bytes = padded[0 .. rows * row_bytes] }, + }); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{mesh} }); + defer store.deinit(); + const source = store.view().source("table", .{ .row, .channel }) + .pad(0, .{ .row = TensorStore.Source.Pad{ .high = @intCast(padded_rows - rows) } }); + const table = switch (placement) { + .rows => source.withPartitioning(.model, .{ .row = .model }).bind(), + .replicated => source.withPartitioning(.model, .{}).bind(), + }; + store.view().setMemory(table, .host_mapped); + try std.testing.expectEqual(@as(i64, @intCast(padded_rows)), table.dim(.row)); + + const live_before = direct_loader.liveHostAllocations(); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 4 }); + defer loader.deinit(); + // The device allocators' counts, once the set-up's calibration has + // allocated on the devices. Bytes in use would not do: CUDA's counts + // what its pool holds, which freed memory leaves room in. CPU's plugin + // counts nothing. + try loader.awaitSetup(); + var allocations: [Platform.MAX_NUM_DEVICES]?u64 = undefined; + for (devices) |device| allocations[device.id] = platform.devices[device.id].memoryStats().num_allocs; + var device_table: Buffer = undefined; + try loader.start(&table, &device_table, &store, .none); + try loader.wait(); + var freed = false; + defer if (!freed) device_table.deinit(); + // The table took no device memory: nothing was allocated on a device. + for (devices) |device| if (allocations[device.id]) |before| { + try std.testing.expectEqual(before, platform.devices[device.id].memoryStats().num_allocs.?); + }; + try std.testing.expectEqual(padded.len, loader.stats.bytes_loaded); + try testing.expectShards(allocator, io, &device_table, padded, .host_mapped); + // A row shard is one allocation per device, a replicated table one. + try std.testing.expectEqual(live_before + if (placement == .rows) shard_count else 1, direct_loader.liveHostAllocations()); + + const first_pointer = try device_table._shards.get(0).opaqueDeviceMemoryDataPointer(api); + for (devices, device_table._shards.constSlice()) |device, shard| { + // Replicas share one allocation; row shards never do. + const pointer = try shard.opaqueDeviceMemoryDataPointer(api); + const shares = placement == .replicated and shard_count > 1; + if (device.id != devices[0].id) try std.testing.expectEqual(shares, pointer == first_pointer); + } + + // Each device gathers `gathered` rows of its own shard, the padding + // rows of the last one included (of every replica, replicated), once + // a chain of matrix products is done: the indices depend on them. + const Gather = struct { + fn forward(table_: Tensor, indices: Tensor, ballast: Tensor) Tensor { + var product = ballast; + for (0..8) |_| product = product.dot(product.withTags(.{ .k, .j }), .k).withTags(.{ .i, .k }); + // Zero, as every product keeps the ballast's positive entries. + const offset: Tensor = .select( + product.sum(.i).sum(.k).reshape(Shape.init(.{}, .f32)).cmp(.LT, .scalar(0, .f32)), + .scalar(1, .u32), + .scalar(0, .u32), + ); + const output: Shape = Shape.init(.{ .shard = indices.dim(.shard), .bytes = indices.dim(.k) * table_.dim(.channel) }, .u8) + .withPartitioning(table_.shape()._sharding.mesh.?, .{ .shard = .model }); + return ops.manualComputation((struct { + fn body(inputs: struct { table: Tensor, indices: Tensor, offset: Tensor }, local: Shape) Tensor { + const shifted = inputs.indices.add(inputs.offset.broad(inputs.indices.shape())); + return inputs.table.gather(.{ .row = shifted }, .{}).reshape(local); + } + }).body, .{ .table = table_, .indices = indices, .offset = offset }, output); + } + }; + const indices_shape = Shape.init(.{ .shard = shard_count, .k = gathered }, .u32).withPartitioning(mesh, .{ .shard = .model }); + const indices = try allocator.alloc(u32, shard_count * gathered); + defer allocator.free(indices); + for (0..shard_count) |shard| { + const local = indices[shard * gathered ..][0..gathered]; + for (local, 0..) |*index, k| index.* = @intCast((k * 7919 + shard * 104_729) % shard_rows); + // The first and last rows of every shard; on the last device these + // are padding. + local[0] = 0; + local[1] = @intCast(shard_rows - 1); + local[2] = @intCast(shard_rows - 2); + } + var indices_buffer: Buffer = try .from(io, platform, indices_shape, std.mem.sliceAsBytes(indices), .{}); + defer indices_buffer.deinit(); + // Milliseconds of products on an accelerator; a token amount on CPU. + const ballast_dim: usize = if (platform.target == .cpu) 32 else 4096; + const ballast_shape = Shape.init(.{ .i = ballast_dim, .k = ballast_dim }, .f32).withPartitioning(mesh, .{}); + const ballast = try allocator.alloc(f32, ballast_dim * ballast_dim); + defer allocator.free(ballast); + @memset(ballast, 1.0 / @as(f32, @floatFromInt(ballast_dim))); + var ballast_buffer: Buffer = try .from(io, platform, ballast_shape, std.mem.sliceAsBytes(ballast), .{}); + defer ballast_buffer.deinit(); + + const exe = try platform.compileFn(allocator, io, Gather.forward, .{ Tensor.fromShape(table.shape()), Tensor.fromShape(indices_shape), Tensor.fromShape(ballast_shape) }, .{ .meshes = &.{mesh} }); + defer exe.deinit(); + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + args.set(.{ device_table, indices_buffer, ballast_buffer }); + try exe.call(io, args, &results, .{}); + var output = results.get(Buffer); + defer output.deinit(); + + // The table goes while the gather may still have to read it. Pinned + // buffers of its size and another pattern would take its memory had it + // gone early, and the gather would return their bytes. + device_table.deinit(); + freed = true; + const decoy_bytes = try allocator.alloc(u8, padded.len); + defer allocator.free(decoy_bytes); + @memset(decoy_bytes, 0xee); + var decoy: Buffer = try .from(io, platform, table.shape(), decoy_bytes, .{ .memory = .host_pinned }); + defer decoy.deinit(); + + const expected = try allocator.alloc(u8, shard_count * gathered * row_bytes); + defer allocator.free(expected); + for (0..shard_count) |shard| { + for (0..gathered) |k| { + const first_row = if (placement == .rows) shard * shard_rows else 0; + const row = first_row + indices[shard * gathered + k]; + @memcpy(expected[(shard * gathered + k) * row_bytes ..][0..row_bytes], padded[row * row_bytes ..][0..row_bytes]); + } + } + try testing.expectShards(allocator, io, &output, expected, .default); + + // The allocations go with their last views, once nothing reads them; + // PJRT may drop the views on its own threads just after the output is + // ready. + var polls: usize = 0; + while (direct_loader.liveHostAllocations() != live_before) : (polls += 1) { + if (polls == 5000) return error.TestUnexpectedResult; + try io.sleep(.fromMilliseconds(1), .awake); + } + + // The control: the same table in device memory is allocated on every + // device, so the check after the load above can fail. + const device_copy = switch (placement) { + .rows => source.withPartitioning(.model, .{ .row = .model }).bind(), + .replicated => source.withPartitioning(.model, .{}).bind(), + }; + for (devices) |device| allocations[device.id] = platform.devices[device.id].memoryStats().num_allocs; + var device_buffer: Buffer = undefined; + try loader.start(&device_copy, &device_buffer, &store, .none); + try loader.wait(); + defer device_buffer.deinit(); + for (devices) |device| if (allocations[device.id]) |before| { + try std.testing.expect(platform.devices[device.id].memoryStats().num_allocs.? > before); + }; +} + +/// A loader on the buffered backend, whichever backend `platform` loads +/// through (`backend.loadsDirect`), its set-up done in place: CPU loads +/// through the direct backend, and this keeps the front end over the +/// buffered one under test. +fn initBuffered(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Loader.Options) !Loader { + const backend_opts = try backendOptions(io, opts); + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .setup = null, + .backend = .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, backend_opts) }, + .program_group_bytes = opts.program_group_bytes, + }; +} + +test "the front end loads through the buffered backend and fails with it" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try initBuffered(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + // A tensor, one assembled from value's first half and second's last + // half, one padded with zeros, and a program tensor, in a group of its + // own. + const Source = TensorStore.Source; + const view = fixture.store.view(); + const Model = struct { value: Tensor, assembled: Tensor, padded: Tensor, computed: Tensor }; + const model: Model = .{ + .value = fixture.value, + .assembled = Source.concatenate(&.{ + view.source("value", .{.x}).slice(.x, .{ .end = 2 }), + view.source("second", .{.x}).slice(.x, .{ .start = 2 }), + }, .x).withPartitioning(.replicated, .{}).bind(), + .padded = view.source("second", .{.x}).pad(0, .{ .x = Source.Pad{ .high = 2 } }).withPartitioning(.replicated, .{}).bind(), + .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Add{ .value = 1 }), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &buffers.assembled, &.{ 1, 2, 7, 8 }, .default); + try testing.expectShards(allocator, io, &buffers.padded, &.{ 5, 6, 7, 8, 0, 0 }, .default); + try testing.expectShards(allocator, io, &buffers.computed, &.{ 2, 3, 4, 5 }, .default); + + // A missing file fails the loader, and every later call with it. + var broken: Buffer = undefined; + try loader.start(&fixture.missing, &broken, &fixture.store, .none); + defer broken.deinit(); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + try std.testing.expectEqual(0, broken._shards.len); + try std.testing.expectError(error.FileNotFound, loader.start(&fixture.value, &broken, &fixture.store, .none)); +} diff --git a/zml/io/log.zig b/zml/io/log.zig new file mode 100644 index 00000000..e5bb5820 --- /dev/null +++ b/zml/io/log.zig @@ -0,0 +1,38 @@ +//! The loader's log scopes. A failure is logged once, where it is +//! detected, at `err` with the values that explain it; cancellation and a +//! plain allocation failure log at `debug`. Frames that only propagate an +//! error do not log it again. Test builds log `err` at `debug`, since the +//! test runner counts logged errors as failures: error-path tests assert +//! the returned errors and the cleanup, not the logs. +const std = @import("std"); +const builtin = @import("builtin"); + +pub const load = Scoped(.@"zml/io/load"); +pub const io = Scoped(.@"zml/io"); +pub const mem = Scoped(.@"zml/mem"); + +fn Scoped(comptime scope: @EnumLiteral()) type { + const base = std.log.scoped(scope); + return struct { + pub const debug = base.debug; + pub const info = base.info; + pub const warn = base.warn; + + /// A terminal failure. The test runner counts logged errors as + /// test failures, so test builds log it at `debug` instead. + pub fn err(comptime fmt: []const u8, args: anytype) void { + if (builtin.is_test) base.debug(fmt, args) else base.err(fmt, args); + } + + /// The first failure of a pipeline, with the error's name appended. + /// Cancellation is expected, and an `OutOfMemory` is a plain + /// allocation failure, returned unlogged, or was logged where it + /// failed (host memory): those log at `debug`. + pub fn failure(e: anyerror, comptime fmt: []const u8, args: anytype) void { + const named_fmt = fmt ++ ": {s}"; + const named_args = args ++ .{@errorName(e)}; + const logged = e == error.Canceled or e == error.Cancelled or e == error.OutOfMemory; + if (logged) debug(named_fmt, named_args) else err(named_fmt, named_args); + } + }; +} diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 25ce26d6..927f95ca 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -24,7 +24,7 @@ pub const ProjectionLayout = enum { concatenated, interleaved }; pub const RoutingWeightPlacement = enum { before_down, after_down }; /// How a backend expects the expert weights to be stored -/// The packer writes this layout and forwardMoe reads it +/// The experts are loaded in this layout and forwardMoe reads it pub const ExpertsLayout = struct { /// Whether the two projections occupy contiguous halves or alternate. gate_up: ProjectionLayout, diff --git a/zml/mem.zig b/zml/mem.zig index a64ae968..596ebe85 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -1,320 +1,9 @@ const std = @import("std"); -const Allocator = std.mem.Allocator; -const Alignment = std.mem.Alignment; -const assert = std.debug.assert; - -const pjrt = @import("pjrt"); const Buffer = @import("buffer.zig").Buffer; -const Device = @import("platform.zig").Device; -const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); -const Platform = @import("platform.zig").Platform; const Tensor = @import("tensor.zig").Tensor; -const log = std.log.scoped(.@"zml/mem"); - -pub const DmaAllocator = union(enum) { - passthrough: std.mem.Allocator, - uib: UninitializedBufferAllocator, - dmam: DmaMapAllocator, - - pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { - return switch (device.platform.target) { - .cuda => .{ .dmam = .init(parent, device.platform) }, - .rocm, .oneapi, .tpu, .furiosa => .{ .uib = .init(device.memory(.host_pinned).?) }, - .cpu, .neuron, .metal => .{ .passthrough = parent }, - }; - } - - pub fn allocator(self: *const DmaAllocator) std.mem.Allocator { - return switch (self.*) { - .passthrough => |a| a, - inline else => |*a| a.allocator(), - }; - } -}; - -pub const UninitializedBufferAllocator = struct { - memory: *const Memory, - - const Header = struct { - buffer: *pjrt.Buffer, - }; - - pub fn init(memory: *const Memory) UninitializedBufferAllocator { - return .{ - .memory = memory, - }; - } - - pub fn allocator(self: *const UninitializedBufferAllocator) std.mem.Allocator { - return .{ - .ptr = @constCast(self), - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }, - }; - } - - fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { - const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); - const pjrt_api = self.memory.platform.pjrt_api; - const pjrt_client = self.memory.platform.pjrt_client; - - const total_len = std.mem.alignForward(usize, @sizeOf(Header) + len, alignment.toByteUnits()); - - const pjrt_buffer = pjrt_client.createUninitializedBuffer(pjrt_api, .{ - .dims = &.{@intCast(total_len)}, - .element_type = .u8, - .layout = .{ - .tiled = .{ - .minor_to_major = &.{0}, - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - .dst = .{ .memory = self.memory.pjrt_memory }, - }) catch return null; - - const opaque_ptr: [*]u8 = @ptrCast(pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable); - const data_with_header: []u8 = opaque_ptr[0..total_len]; - - const header = std.mem.bytesAsValue(Header, opaque_ptr); - header.* = .{ - .buffer = pjrt_buffer, - }; - const offset = std.mem.alignForward(usize, @sizeOf(Header), alignment.toByteUnits()); - return @ptrCast(data_with_header[offset..]); - } - - fn free(ctx: *anyopaque, buf: []u8, alignment: Alignment, ret_addr: usize) void { - _ = ret_addr; - const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); - const pjrt_api = self.memory.platform.pjrt_api; - const header: *Header = @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(buf.ptr) - @sizeOf(Header), alignment.toByteUnits())); - header.buffer.deinit(pjrt_api); - } - - fn resize(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return false; - } - - fn remap(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return null; - } -}; - -pub const DmaMapAllocator = struct { - parent: std.mem.Allocator, - platform: *const Platform, - - pub fn init(parent: std.mem.Allocator, platform: *const Platform) DmaMapAllocator { - return .{ - .parent = parent, - .platform = platform, - }; - } - - pub fn allocator(self: *const DmaMapAllocator) Allocator { - return .{ - .ptr = @constCast(self), - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }, - }; - } - - fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { - const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); - const allocation = self.parent.rawAlloc(len, alignment, ret_addr); - if (allocation) |loc| { - const data = loc[0..len]; - self.platform.pjrt_client.dmaMap(self.platform.pjrt_api, @ptrCast(data)) catch { - self.parent.rawFree(data, alignment, ret_addr); - return null; - }; - } - return allocation; - } - - fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return false; - } - - fn remap(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return null; - } - - fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { - const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); - self.platform.pjrt_client.dmaUnmap(self.platform.pjrt_api, @ptrCast(buf[0..buf.len])) catch unreachable; - self.parent.rawFree(buf, alignment, ret_addr); - } -}; - -pub const FixedBufferPool = struct { - buffer: []u8, - block_size: usize, - q_buf: []const u16, - q: std.Io.Queue(u16), - - pub fn init(allocator: std.mem.Allocator, buffer_: []u8, blocks_: u16) !FixedBufferPool { - const block_size = @divExact(buffer_.len, blocks_); - const q_buf = try allocator.alloc(u16, blocks_); - for (q_buf, 0..) |*idx, i| { - idx.* = @intCast(i); - } - var q: std.Io.Queue(u16) = .init(q_buf); - q.type_erased.len = q.type_erased.buffer.len; // make the queue full - return .{ - .buffer = buffer_, - .block_size = block_size, - .q = q, - .q_buf = q_buf, - }; - } - - pub fn deinit(self: *FixedBufferPool, allocator: std.mem.Allocator) void { - allocator.free(self.q_buf); - } - - pub fn get(self: *FixedBufferPool, io: std.Io) ![]u8 { - const idx = try self.q.getOneUncancelable(io); - return self.buffer[idx * self.block_size ..][0..self.block_size]; - } - - fn inRange(sub_buffer: []const u8, buffer: []const u8) bool { - return @intFromPtr(sub_buffer.ptr) >= @intFromPtr(buffer.ptr) and - @intFromPtr(sub_buffer[sub_buffer.len - 1 ..].ptr) <= @intFromPtr(buffer[buffer.len - 1 ..].ptr); - } - - pub fn put(self: *FixedBufferPool, io: std.Io, buf: []u8) void { - // is the pointer in range ? - std.debug.assert(inRange(buf, self.buffer)); - const idx = @divExact(@intFromPtr(buf.ptr) - @intFromPtr(self.buffer.ptr), self.block_size); - self.q.putOneUncancelable(io, @intCast(idx)) catch unreachable; - } -}; - -pub const DynamicBufferPool = struct { - const Node = struct { next: ?*Node }; - const alignment: std.mem.Alignment = .of(Node); - - block_size: usize, - max_blocks: usize, - - arena_state: std.heap.ArenaAllocator.State = .{}, - free_stack: std.atomic.Value(?*Node) = .init(null), - in_flight: std.atomic.Value(usize) = .init(0), - mutex: std.Io.Mutex = .init, - condition: std.Io.Condition = .init, - - pub fn init(max_blocks: usize, block_size: usize) DynamicBufferPool { - return .{ - .block_size = block_size, - .max_blocks = max_blocks, - }; - } - - pub fn deinit(self: *DynamicBufferPool, allocator: std.mem.Allocator) void { - self.arena_state.promote(allocator).deinit(); - } - - pub fn get(self: *DynamicBufferPool, allocator: std.mem.Allocator, io: std.Io) ![]u8 { - while (true) { - var in_flight = self.in_flight.load(.acquire); - while (in_flight < self.max_blocks) { - if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { - in_flight = actual; - continue; - } - - if (self.pop()) |node| { - return self.nodeToSlice(node); - } - - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - var arena = self.arena_state.promote(allocator); - defer self.arena_state = arena.state; - - errdefer _ = self.in_flight.fetchSub(1, .release); - const buffer = try arena.allocator().alignedAlloc(u8, alignment, self.block_size); - return @alignCast(buffer); - } - - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - while (self.in_flight.load(.acquire) >= self.max_blocks) { - try self.condition.wait(io, &self.mutex); - } - } - } - - pub fn put(self: *DynamicBufferPool, io: std.Io, buf: []u8) void { - const node: *Node = @ptrCast(@alignCast(buf.ptr)); - var head = self.free_stack.load(.acquire); - while (true) { - node.next = head; - if (self.free_stack.cmpxchgWeak(head, node, .release, .acquire)) |actual_head| { - head = actual_head; - } else { - break; - } - } - - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - _ = self.in_flight.fetchSub(1, .release); - self.condition.signal(io); - } - - fn pop(self: *DynamicBufferPool) ?*Node { - var head = self.free_stack.load(.acquire); - while (true) { - if (self.free_stack.cmpxchgWeak(head, if (head) |h| h.next else null, .acquire, .acquire)) |actual| { - head = actual; - continue; - } - - return head; - } - } - - fn nodeToSlice(self: DynamicBufferPool, node: *Node) []u8 { - const ptr: [*]u8 = @ptrCast(node); - return ptr[0..self.block_size]; - } -}; - /// Return a clone of a type with Tensors replaced by Buffer. /// Non-Tensor metadata is stripped out of the resulting struct. /// Recursively descends into the type. @@ -323,6 +12,21 @@ pub fn Bufferized(comptime T: type) type { return meta.MapRestrict(Tensor, Buffer).map(T); } +/// The field of `Bufferized(Model)` that holds the struct `Model`'s field +/// `name`, or null when `Bufferized` strips it. `Bufferized` keeps the +/// fields holding tensors, in order, and renumbers a tuple's. +fn bufferizedField(comptime Model: type, comptime name: []const u8) ?[]const u8 { + @setEvalBranchQuota(10_000); + const info = @typeInfo(Model).@"struct"; + var index: usize = 0; + for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { + if (attrs.@"comptime" or !meta.Contains(Field, Tensor)) continue; + if (std.mem.eql(u8, field_name, name)) return @typeInfo(Bufferized(Model)).@"struct".field_names[index]; + index += 1; + } + return null; +} + fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bufferized(@TypeOf(model))) !void { @setEvalBranchQuota(10_000); const Model = @TypeOf(model); @@ -341,8 +45,16 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const type_info = @typeInfo(ModelBufferized); switch (type_info) { .@"struct" => |struct_type_info| { - inline for (struct_type_info.field_names) |field_name| { - try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); + var initialized_fields: usize = 0; + errdefer inline for (struct_type_info.field_names, 0..) |field_name, index| { + if (index < initialized_fields) + deinitBufferized(allocator, &@field(bufferized_, field_name)); + }; + inline for (@typeInfo(Model).@"struct".field_names) |field_name| { + if (comptime bufferizedField(Model, field_name)) |buffer_field| { + try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, buffer_field)); + initialized_fields += 1; + } } }, .@"union" => { @@ -364,23 +76,333 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); - for (model, bufferized_.*) |src, *dst| { - try bufferizeInner(allocator, src, dst); + const allocated = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); + var initialized: usize = 0; + errdefer { + for (allocated[0..initialized]) |*element| deinitBufferized(allocator, element); + allocator.free(allocated); } + for (model, allocated) |src, *dst| { + try bufferizeInner(allocator, src, dst); + initialized += 1; + } + bufferized_.* = allocated; }, else => unreachable, } }, + .array => |info| { + var initialized: usize = 0; + errdefer for (bufferized_.*[0..initialized]) |*element| deinitBufferized(allocator, element); + inline for (0..info.len) |index| { + try bufferizeInner(allocator, model[index], &bufferized_.*[index]); + initialized = index + 1; + } + }, .void, .int, .@"enum", .bool, .enum_literal, .float, .vector => {}, else => unreachable, } } -/// Convert a model to its bufferized form by replacing Tensor fields with Buffer -/// and allocating any required slices using the provided allocator. -pub inline fn bufferize(allocator: std.mem.Allocator, comptime ModelType: type, model: *const ModelType) !Bufferized(ModelType) { - var bufferized: Bufferized(ModelType) = undefined; +/// Convert the model `model` points to into its bufferized form by replacing +/// Tensor fields with Buffer and allocating any required slices using the +/// provided allocator. +pub inline fn bufferize(allocator: std.mem.Allocator, model: anytype) !Bufferized(@TypeOf(model.*)) { + var bufferized: Bufferized(@TypeOf(model.*)) = undefined; try bufferizeInner(allocator, model.*, &bufferized); return bufferized; } + +/// A tensor of a model, the buffer at its place in the model's buffers, and +/// its field path from the root, as `.layers[1].ffn.weight`. +pub const Pair = struct { tensor: *const Tensor, buffer: *Buffer, path: []const u8 }; + +/// Pairs each tensor of the model `model` points to with the buffer at its +/// place in `bufferized`, in the order `meta.visit` finds the tensors. The +/// pairs and their paths live in `arena`. The two trees must have the +/// structure `bufferize` gives: the same union arms, the same optionals set, +/// slices of the same lengths. `report`, an empty fixed writer, builds the +/// paths and cuts them short when full. At the first place the trees differ +/// the walk stops and returns `error.Mismatch`, and `report` ends with the +/// path there and what differs. +pub fn pairBufferized( + arena: std.mem.Allocator, + model: anytype, + bufferized: *Bufferized(@TypeOf(model.*)), + report: *std.Io.Writer, +) error{ Mismatch, OutOfMemory }![]Pair { + var pairs: std.ArrayList(Pair) = try .initCapacity(arena, meta.count(Tensor, model)); + try pairBufferizedInner(arena, model, bufferized, report, &pairs); + return pairs.items; +} + +/// `report.end` is restored after each part that pairs, so at a tensor or a +/// mismatch the report holds the path down to it. +fn pairBufferizedInner( + arena: std.mem.Allocator, + model: anytype, + bufferized: anytype, + report: *std.Io.Writer, + pairs: *std.ArrayList(Pair), +) error{ Mismatch, OutOfMemory }!void { + const Model = @typeInfo(@TypeOf(model)).pointer.child; + if (comptime !meta.Contains(Model, Tensor)) return; + if (Model == Tensor) return pairs.appendAssumeCapacity(.{ + .tensor = model, + // `bufferize` allocates every slice mutable, a `[]const` one included. + .buffer = @constCast(bufferized), + .path = try arena.dupe(u8, report.buffered()), + }); + + const depth = report.end; + switch (@typeInfo(Model)) { + .@"struct" => |info| inline for (info.field_names) |name| { + if (comptime bufferizedField(Model, name)) |buffer_field| { + report.print(".{s}", .{name}) catch {}; + try pairBufferizedInner(arena, &@field(model, name), &@field(bufferized, buffer_field), report, pairs); + report.end = depth; + } + }, + .@"union" => { + const tag = std.meta.activeTag(model.*); + if (tag != std.meta.activeTag(bufferized.*)) { + report.print(": union arm .{t} in the model, .{t} in the buffers", .{ tag, std.meta.activeTag(bufferized.*) }) catch {}; + return error.Mismatch; + } + switch (model.*) { + inline else => |*payload, arm| { + report.print(".{s}", .{@tagName(arm)}) catch {}; + try pairBufferizedInner(arena, payload, &@field(bufferized, @tagName(arm)), report, pairs); + report.end = depth; + }, + } + }, + .optional => { + if ((model.* == null) != (bufferized.* == null)) { + report.print(": {s} in the model, {s} in the buffers", .{ if (model.* == null) "null" else "set", if (bufferized.* == null) "null" else "set" }) catch {}; + return error.Mismatch; + } + if (model.*) |*payload| try pairBufferizedInner(arena, payload, &bufferized.*.?, report, pairs); + }, + .pointer => |info| { + if (info.size != .slice) @compileError("mem.pairBufferized: " ++ @typeName(Model) ++ " is not a slice"); + if (model.len != bufferized.len) { + report.print(": a slice of {d} in the model, of {d} in the buffers", .{ model.len, bufferized.len }) catch {}; + return error.Mismatch; + } + for (model.*, bufferized.*, 0..) |*element, *buffer, i| { + report.print("[{d}]", .{i}) catch {}; + try pairBufferizedInner(arena, element, buffer, report, pairs); + report.end = depth; + } + }, + .array => for (model, bufferized, 0..) |*element, *buffer, i| { + report.print("[{d}]", .{i}) catch {}; + try pairBufferizedInner(arena, element, buffer, report, pairs); + report.end = depth; + }, + else => @compileError("mem.pairBufferized: cannot walk " ++ @typeName(Model)), + } +} + +/// Deinitializes every accelerator buffer of the tree `value` points to and +/// frees the recursive slice storage allocated by `bufferize`. +pub fn deinitBufferized(allocator: std.mem.Allocator, value: anytype) void { + const Ptr = @TypeOf(value); + const T = @typeInfo(Ptr).pointer.child; + if (T == Tensor) @compileError("mem.deinitBufferized takes the buffers bufferize returns, not the model"); + if (T == Buffer) { + const buffer: *Buffer = @constCast(value); + buffer.deinit(); + return; + } + switch (@typeInfo(T)) { + .@"struct" => |info| inline for (info.field_names) |field_name| { + deinitBufferized(allocator, &@field(value, field_name)); + }, + .@"union" => switch (value.*) { + inline else => |*payload| deinitBufferized(allocator, payload), + }, + .optional => if (value.*) |*payload| { + deinitBufferized(allocator, payload); + }, + .pointer => |info| switch (info.size) { + .slice => { + for (value.*) |*element| deinitBufferized(allocator, element); + allocator.free(value.*); + }, + else => @compileError("mem.deinitBufferized takes the buffers bufferize returns, not " ++ @typeName(T)), + }, + .array => for (&value.*) |*element| deinitBufferized(allocator, element), + // What `Bufferized` makes of a union arm or a whole tree without a tensor. + .void => {}, + else => @compileError("mem.deinitBufferized takes the buffers bufferize returns, not " ++ @typeName(T)), + } +} + +test "bufferize maps a tuple's fields as Bufferized renumbers them" { + const Model = struct { u32, Tensor, []Tensor, [2]Tensor }; + const a: Tensor = .init(.{2}, .f32); + const b: Tensor = .init(.{3}, .f16); + var tensors = [_]Tensor{ a, b }; + const model: Model = .{ 3, a, &tensors, .{ b, a } }; + var bufferized = try bufferize(std.testing.allocator, &model); + defer deinitBufferized(std.testing.allocator, &bufferized); + try std.testing.expect(bufferized[0].shape().eql(a.shape())); + try std.testing.expectEqual(2, bufferized[1].len); + try std.testing.expect(bufferized[1][1].shape().eql(b.shape())); + try std.testing.expect(bufferized[2][0].shape().eql(b.shape())); + try std.testing.expect(bufferized[2][1].shape().eql(a.shape())); +} + +test "bufferize rolls back earlier slice fields on allocation failure" { + const Model = struct { + first: []const Tensor, + second: []const Tensor, + }; + var tensors: [1]Tensor = undefined; + const model: Model = .{ .first = &tensors, .second = &tensors }; + var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 1 }); + + if (bufferize(failing.allocator(), &model)) |result| { + var unexpected = result; + deinitBufferized(failing.allocator(), &unexpected); + return error.ExpectedOutOfMemory; + } else |err| { + try std.testing.expect(err == error.OutOfMemory); + } + try std.testing.expect(failing.has_induced_failure); + try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); +} + +test "deinitBufferized frees recursive const slices" { + const Layer = struct { weights: []const Tensor }; + const Model = struct { + layers: []const Layer, + fixed: [2]Tensor, + }; + var weights: [2]Tensor = undefined; + const layers = [_]Layer{ + .{ .weights = weights[0..1] }, + .{ .weights = weights[1..2] }, + }; + const model: Model = .{ .layers = &layers, .fixed = undefined }; + var bufferized = try bufferize(std.testing.allocator, &model); + deinitBufferized(std.testing.allocator, &bufferized); +} + +test "deinitBufferized frees slices under optionals and unions" { + const Layer = struct { weights: []const Tensor }; + const Ffn = union(enum) { dense: Layer, experts: []const Layer, none: void }; + const Model = struct { + present: ?Layer, + absent: ?Layer, + ffns: [3]Ffn, + }; + var weights: [3]Tensor = undefined; + const experts = [_]Layer{ .{ .weights = weights[0..1] }, .{ .weights = weights[1..3] } }; + const model: Model = .{ + .present = .{ .weights = &weights }, + .absent = null, + .ffns = .{ .{ .dense = .{ .weights = weights[2..3] } }, .{ .experts = &experts }, .none }, + }; + var bufferized = try bufferize(std.testing.allocator, &model); + deinitBufferized(std.testing.allocator, &bufferized); +} + +test "pairBufferized pairs tensors and buffers in visit order" { + const Layer = struct { weights: []Tensor, scale: ?Tensor }; + const Ffn = union(enum) { dense: Layer, experts: []Layer, none: void }; + const Model = struct { ffns: [3]Ffn, head: Tensor, layers: u32 }; + var weights: [4]Tensor = undefined; + var experts = [_]Layer{ .{ .weights = weights[1..2], .scale = null }, .{ .weights = weights[2..4], .scale = weights[3] } }; + var model: Model = .{ + .ffns = .{ .{ .dense = .{ .weights = weights[0..1], .scale = weights[0] } }, .{ .experts = &experts }, .none }, + .head = undefined, + .layers = 2, + }; + var bufferized = try bufferize(std.testing.allocator, &model); + defer deinitBufferized(std.testing.allocator, &bufferized); + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + var report_buffer: [64]u8 = undefined; + var report: std.Io.Writer = .fixed(&report_buffer); + const pairs = try pairBufferized(arena.allocator(), &model, &bufferized, &report); + try std.testing.expectEqual(0, report.end); + + const Visited = struct { + tensors: [8]*const Tensor = undefined, + buffers: [8]*Buffer = undefined, + tensor_count: usize = 0, + buffer_count: usize = 0, + }; + var visited: Visited = .{}; + meta.forEachVisit(&model, *const Tensor, struct { + fn call(_: usize, tensor: *const Tensor, out: *Visited) void { + out.tensors[out.tensor_count] = tensor; + out.tensor_count += 1; + } + }.call, .{&visited}); + meta.forEachVisit(&bufferized, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, out: *Visited) void { + out.buffers[out.buffer_count] = buffer; + out.buffer_count += 1; + } + }.call, .{&visited}); + const paths = [_][]const u8{ + ".ffns[0].dense.weights[0]", + ".ffns[0].dense.scale", + ".ffns[1].experts[0].weights[0]", + ".ffns[1].experts[1].weights[0]", + ".ffns[1].experts[1].weights[1]", + ".ffns[1].experts[1].scale", + ".head", + }; + try std.testing.expectEqual(paths.len, pairs.len); + try std.testing.expectEqual(pairs.len, visited.tensor_count); + try std.testing.expectEqual(pairs.len, visited.buffer_count); + // Not `expectEqualSlices`: printing a `*Buffer` reaches code a CPU test build lacks. + for (pairs, paths, 0..) |pair, path, i| { + try std.testing.expect(pair.tensor == visited.tensors[i]); + try std.testing.expect(pair.buffer == visited.buffers[i]); + try std.testing.expectEqualStrings(path, pair.path); + } +} + +test "pairBufferized stops at the first place the trees differ and names it" { + const Layer = struct { weights: []Tensor, scale: ?Tensor }; + const Ffn = union(enum) { dense: Layer, none: void }; + const Model = struct { ffns: []Ffn, head: Tensor }; + var weights: [3]Tensor = undefined; + var built = [_]Ffn{ .{ .dense = .{ .weights = weights[0..2], .scale = null } }, .none }; + const built_model: Model = .{ .ffns = &built, .head = undefined }; + var bufferized = try bufferize(std.testing.allocator, &built_model); + defer deinitBufferized(std.testing.allocator, &bufferized); + + const Case = struct { ffns: [2]Ffn, report: []const u8 }; + const cases = [_]Case{ + .{ .ffns = .{ .none, .none }, .report = ".ffns[0]: union arm .none in the model, .dense in the buffers" }, + .{ + .ffns = .{ .{ .dense = .{ .weights = weights[0..2], .scale = weights[2] } }, .none }, + .report = ".ffns[0].dense.scale: set in the model, null in the buffers", + }, + .{ + .ffns = .{ .{ .dense = .{ .weights = weights[0..1], .scale = null } }, .none }, + .report = ".ffns[0].dense.weights: a slice of 1 in the model, of 2 in the buffers", + }, + }; + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + for (cases) |case| { + var ffns = case.ffns; + const model: Model = .{ .ffns = &ffns, .head = undefined }; + var report_buffer: [128]u8 = undefined; + var report: std.Io.Writer = .fixed(&report_buffer); + // Not `expectError`: printing a `*Buffer` reaches code a CPU test build lacks. + if (pairBufferized(arena.allocator(), &model, &bufferized, &report)) |_| { + return error.TestExpectedMismatch; + } else |err| try std.testing.expectEqual(error.Mismatch, err); + try std.testing.expectEqualStrings(case.report, report.buffered()); + } +} diff --git a/zml/meta.zig b/zml/meta.zig index 1eefea11..504b2973 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -332,6 +332,13 @@ pub fn MapRestrict(From: type, To: type) type { if (!Contains(T, From)) return void; + // `visit` walks a bounded array's `len` elements, but its mapped + // struct would hold the whole buffer: pairing the two by visit + // order would miss or misplace elements. + if (@typeInfo(T) == .@"struct" and @hasDecl(T, "constSlice") and @hasDecl(T, "slice")) { + stdx.debug.compileError("zml.meta.MapRestrict({any}, {any}) cannot map the bounded array {any}: use a slice or an array", .{ From, To, T }); + } + return switch (@typeInfo(T)) { .@"struct" => |struct_infos| { // We know that at least one of the struct field contains a From. diff --git a/zml/nn.zig b/zml/nn.zig index 56fd8b51..e8d9873d 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -31,10 +31,6 @@ pub const Linear = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { - zml.Buffer.deinitAll(Linear, self); - } - pub fn quantizationScheme(self: Linear) ?Quantization.Scheme { return if (self.quantization) |q| q.scheme else null; } @@ -227,7 +223,7 @@ test "block128 scaled dot layouts" { const platform = zml.testing.env(); // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) if (platform.target != .cuda) return error.SkipZigTest; - const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; + const dtype = Quantization.Target.of(platform).fp8.?; const Local = struct { const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 824d2bd1..bfa4fc7d 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -14,7 +14,11 @@ const log = std.log.scoped(.@"zml/pjrtx"); pub const Client = opaque { pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || pjrt.ApiError; - pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { + /// Compiles on the calling thread: under zio, `Compiler.compile` has + /// already moved off the coroutine onto a pool worker of its own + /// (`zio.blockInPlaceReserved`), and XLA's compile needs the thread's + /// stack, not a task's. + pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { break :blk stablehlo.smallerVersion(requested_version, stablehlo.currentVersion()); @@ -37,7 +41,7 @@ pub const Client = opaque { }; }; - return @ptrCast(try client.compile(api, io, .{ + return @ptrCast(try client.compileRaw(api, .{ .bytecode = serialized_buffer.written(), .bytecode_format = .mlir, .compile_options_pb = compile_options_pb, diff --git a/zml/platform.zig b/zml/platform.zig index 3b52c08e..6f9d114d 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_rocm = @import("platforms/rocm"); const stdx = @import("stdx"); pub const Target = @import("platforms").Platform; @@ -37,6 +38,10 @@ fn validateDeviceCount(target: Target, num_devices: usize) !void { log.err("The selected platform requires at least 1 device, got {}", .{num_devices}); return error.MissingDevices; } + if (num_devices > Platform.MAX_NUM_DEVICES) { + log.err("Platform {} got {} devices, but ZML supports up to {}", .{ target, num_devices, Platform.MAX_NUM_DEVICES }); + return error.InvalidDeviceCount; + } switch (target) { .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi, .furiosa => { if (!std.math.isPowerOfTwo(num_devices)) { @@ -300,6 +305,16 @@ pub const Platform = struct { meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), replicated_mesh: *const zml.Mesh, io_impl: IoImpl, + /// Whether `releaseFreedDeviceMemory` asks the devices to give freed + /// memory back. `init` sets it for XLA's VMM allocator on CUDA and ROCm + /// (`CreateOptions.XlaGpu.Allocator.vmm`), which keeps freed memory + /// mapped, and counted in use, until it is reused or reclaimed. Null + /// otherwise: other allocators free at once, and oneAPI's VMM allocator + /// is not proven to follow the same rule. The flag lives in the + /// platform's arena, not in `Platform`: the first device that does not + /// answer as the VMM allocator does clears it for good, through a + /// `*const Platform`, without writing to the platform. + releases_freed_memory: ?*std.atomic.Value(bool) = null, pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; @@ -313,11 +328,9 @@ pub const Platform = struct { var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); + errdefer pjrt_client.deinit(api); const pjrt_devices = pjrt_client.addressableDevices(api); try validateDeviceCount(target, pjrt_devices.len); - if (pjrt_devices.len > MAX_NUM_DEVICES) { - log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); - } const pjrt_memories = pjrt_client.addressableMemories(api); @@ -351,7 +364,18 @@ pub const Platform = struct { const arena = platform.arena.allocator(); errdefer platform.arena.deinit(); + errdefer if (comptime Target.cuda.isEnabled()) platform.state.deinit(); try platform.meshes.ensureTotalCapacity(arena, 8); + const keeps_freed_memory = switch (target) { + .cuda => options.cuda.allocator == .vmm, + .rocm => options.rocm.allocator == .vmm, + else => false, + }; + if (keeps_freed_memory) { + const releases = try arena.create(std.atomic.Value(bool)); + releases.* = .init(true); + platform.releases_freed_memory = releases; + } { const devices = try arena.alloc(Device, pjrt_devices.len); @@ -424,6 +448,117 @@ pub const Platform = struct { } else error.Unavailable; } + /// Forces lazy device allocators to materialize their backing pools. + /// + /// In particular, a preallocated GPU BFC allocator does not reserve its + /// fixed arena until its first non-empty allocation. The temporary buffer + /// is released immediately; caching allocators retain their initialized + /// pools for subsequent allocations. Calling this more than once is safe. + /// Devices are warmed concurrently: a serial pass over eight GPUs costs + /// measurable milliseconds ahead of a load. A device whose task `io` + /// cannot spawn is warmed in place. + pub fn warmupDeviceAllocators(self: *const Platform, io: std.Io) !void { + const Worker = struct { + platform: *const Platform, + device_index: usize, + first_error: *stdx.FirstError(pjrt.ApiError), + + fn run(worker: @This()) void { + const dims: []const i64 = &.{1}; + const memory = worker.platform.devices[worker.device_index].memory(.default).?; + const buffer = worker.platform.pjrt_client.createUninitializedBuffer(worker.platform.pjrt_api, .{ + .dims = dims, + .element_type = pjrtx.bufferTypeFromDtype(.u8), + .layout = worker.platform.defaultMemoryLayout(dims, .u8), + .dst = .{ .memory = memory.pjrt_memory }, + }) catch |err| { + _ = worker.first_error.set(err); + return; + }; + buffer.deinit(worker.platform.pjrt_api); + } + }; + var first_error: stdx.FirstError(pjrt.ApiError) = .{}; + var group: std.Io.Group = .init; + for (self.devices, 0..) |_, device_index| { + const worker: Worker = .{ .platform = self, .device_index = device_index, .first_error = &first_error }; + group.concurrent(io, Worker.run, .{worker}) catch worker.run(); + } + try group.await(io); + if (first_error.get()) |err| return err; + } + + /// Gives back to the devices memory that nothing uses but that stays + /// held: the freed buffers XLA's VMM allocator keeps, which + /// `Device.memoryStats` counts in `bytes_in_use`, and on ROCm what the + /// HIP runtime keeps. + /// + /// XLA's VMM allocator keeps a freed allocation mapped, and counted in + /// use, until a request of the same size reuses it, a request beyond + /// the device's limit reclaims it, or an executable with remapped + /// command buffers is destroyed. A request one byte beyond the limit + /// completes the deferred frees of the device's memory, once its + /// compute stream reached them, then fails with `ResourceExhausted`, as + /// expected. It leaves out the frees of collective memory, which stay + /// counted, and of pinned host memory, which `bytes_in_use` does not + /// count. Any other answer means the plugin does not follow that rule: + /// it is logged once, naming the device, and clears + /// `releases_freed_memory`, so later calls skip this part. This part is + /// a no-op for other allocators, which free at once, and for a device + /// that reports no limit. + /// + /// On ROCm, whatever the allocator, it first makes the HIP runtime give + /// back the device memory it keeps outside the allocator, chiefly the + /// scratch memory of kernels that spill registers, which `memoryStats` + /// never counts but the devices do + /// (`platforms_rocm.releaseRuntimeDeviceMemory`). That part reaches + /// every queue the process has on these devices, other platforms' + /// included, and makes running work wait: each queue holding scratch is + /// suspended and resumed, the call waits for its kernels using scratch, + /// and allocations on the device stall meanwhile. A call within a load + /// can overlap the load's other compiles and transfers, which then wait + /// too. A missing HIP runtime, or a device that does not refuse, is + /// logged once and ends that part for the process. + pub fn releaseFreedDeviceMemory(self: *const Platform) void { + if (comptime Target.rocm.isEnabled()) { + if (self.target == .rocm) _ = self.releaseRocmRuntimeMemory(); + } + const releases = self.releases_freed_memory orelse return; + for (self.devices) |*device| { + if (!releases.load(.monotonic)) return; + const limit = device.memoryStats().bytes_limit orelse continue; + // A limit that large is no limit: nothing goes beyond it. + const beyond_limit = std.math.cast(i64, limit + 1) orelse continue; + const dims: []const i64 = &.{beyond_limit}; + const answer: []const u8 = if (self.pjrt_client.createUninitializedBufferSilent(self.pjrt_api, .{ + .dims = dims, + .element_type = pjrtx.bufferTypeFromDtype(.u8), + .layout = self.defaultMemoryLayout(dims, .u8), + .dst = .{ .memory = device.memory(.default).?.pjrt_memory }, + })) |buffer| answer: { + buffer.deinit(self.pjrt_api); + break :answer "a buffer"; + } else |err| switch (err) { + error.ResourceExhausted => continue, + else => @errorName(err), + }; + if (releases.swap(false, .monotonic)) { + log.warn("Device {d} answered a request beyond its memory limit with {s}, not ResourceExhausted; freed device memory is no longer given back and may stay counted as in use", .{ device.id(), answer }); + } + return; + } + } + + /// `platforms_rocm.releaseRuntimeDeviceMemory` over every device: a + /// device's HIP ordinal is its local hardware id. + fn releaseRocmRuntimeMemory(self: *const Platform) bool { + var ordinals: [MAX_NUM_DEVICES]c_int = undefined; + for (self.devices, ordinals[0..self.devices.len]) |*device, *ordinal| { + ordinal.* = device.localHardwareId(); + } + return platforms_rocm.releaseRuntimeDeviceMemory(ordinals[0..self.devices.len]); + } + pub fn formatWithAttributes(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { const tee = "├─ "; const line = "│ "; @@ -603,6 +738,14 @@ pub const Platform = struct { try writer.writeAll(" }"); } + /// `shape` placed as a buffer places it: on its mesh, or replicated on + /// every device (`replicated_mesh`) when it names none. + pub fn placedShape(self: *const Platform, shape: zml.Shape) zml.Shape { + var placed = shape; + if (placed._sharding.mesh == null) placed._sharding.mesh = self.replicated_mesh; + return placed; + } + pub fn memoryKind(self: *const Platform, kind: Memory.Kind) []const u8 { for (self.memories) |mem| { if (mem.isOfKind(kind)) { @@ -1064,3 +1207,63 @@ test "platform defaultMemoryLayout is boring" { }); } } + +test "platform device allocators can be warmed repeatedly" { + const platform = zml.testing.env(); + + try platform.warmupDeviceAllocators(std.testing.io); + try platform.warmupDeviceAllocators(std.testing.io); +} + +test "platform ROCm devices refuse a request for their whole memory, then still work" { + if (comptime !Target.rocm.isEnabled()) return error.SkipZigTest; + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + // Every device refuses a request for its whole memory, the request ROCr + // answers by reclaiming its queues' scratch and freeing its caches. The + // release itself is not visible from here: HIP counts ROCr's caches as + // free, and only the driver's counters in sysfs show it. + try std.testing.expect(platform.releaseRocmRuntimeMemory()); + + // The devices still take and give back data afterwards. + const values = [_]u32{ 1, 2, 3, 4, 5, 6, 7, 8 }; + var buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, .init(.{ .n = values.len }, .u32), std.mem.sliceAsBytes(&values)); + defer buffer.deinit(); + const read: zml.Slice = try buffer.toSliceAlloc(std.testing.allocator, std.testing.io); + defer read.free(std.testing.allocator); + try std.testing.expectEqualSlices(u32, &values, read.constItems(u32)); +} + +test "platform gives freed device memory back" { + const platform = zml.testing.env(); + // On GPU CI, CUDA and ROCm run this with XLA's VMM allocator. + const releases = platform.releases_freed_memory orelse return error.SkipZigTest; + const device = &platform.devices[0]; + // Without a limit the drain does nothing. + try std.testing.expect(device.memoryStats().bytes_limit != null); + + // An odd size, so that no request of the same size reuses it once freed. + const size = 64 * 1024 * 1024 + 4096; + const dims: []const i64 = &.{size}; + const buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, .{ + .dims = dims, + .element_type = pjrtx.bufferTypeFromDtype(.u8), + .layout = platform.defaultMemoryLayout(dims, .u8), + .dst = .{ .memory = device.memory(.default).?.pjrt_memory }, + }); + { + // Once its allocation is ready, destroying it frees it at once, + // not later on another thread. + errdefer buffer.deinit(platform.pjrt_api); + const ready = buffer.readyEvent(platform.pjrt_api); + defer ready.deinit(platform.pjrt_api); + try ready.await(platform.pjrt_api, std.testing.io); + } + buffer.deinit(platform.pjrt_api); + const in_use_freed = device.memoryStats().bytes_in_use; + + platform.releaseFreedDeviceMemory(); + try std.testing.expect(releases.load(.monotonic)); + try std.testing.expect(device.memoryStats().bytes_in_use + size <= in_use_freed); +} diff --git a/zml/quantization.zig b/zml/quantization.zig index 89e1b210..9053b26c 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -105,6 +105,38 @@ pub const Quantization = struct { return self == .mxfp8 or self == .mxfp4; } }; + + /// What a platform's FP8 GEMMs read, so block-FP8 weights can be bound + /// for it once. Binding them requires the store to carry one + /// (`zml.io.TensorStore.Options.quantization`); the code that binds them + /// reads it (`TensorStore.View.quantization`) and converts or lays them + /// out while binding. + pub const Target = struct { + /// The E4M3 encoding of FP8 weights and activations: FNUZ on CDNA3, + /// FN on CUDA, CDNA4 and RDNA4. Null on other platforms: block-FP8 + /// activations stay BF16 (`quantizeInput`), and weights keep the + /// checkpoint's encoding. + fp8: ?DataType = .f8e4m3fn, + /// Whether e8m0 block scales are laid out in the 128x4 blocks of + /// Blackwell's block-scaled MMA (CUDA sm_10x). + swizzled_scales: bool = false, + + /// `platform`'s target: on CUDA, FN, its scales swizzled on sm_10x; + /// on ROCm, by architecture, null when its compute capability is + /// unknown; elsewhere, CPU included, null and unswizzled. The only + /// platform switch for block-FP8 weights. + pub fn of(platform: *const Platform) Target { + return switch (platform.target) { + .cuda => .{ .swizzled_scales = if (platform_mod.cuda.computeCapability(platform)) |cc| cc.major == 10 else false }, + .rocm => .{ .fp8 = if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, + } else null }, + else => .{ .fp8 = null }, + }; + } + }; }; pub const QuantizedInput = struct { @@ -127,16 +159,7 @@ pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, return switch (quantization.scheme) { .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input.convert(.bf16), global_scale, axis) else null, .fp8_block32, .fp8_block128 => blk: { - const dtype: DataType = switch (platform.target) { - .cuda => .f8e4m3fn, - .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => .f8e4m3fn, - .cdna3 => .f8e4m3fnuz, - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => return null, - } else return null, - else => return null, - }; - + const dtype = Quantization.Target.of(platform).fp8 orelse return null; break :blk switch (quantization.scheme) { .fp8_block32 => quantizeBlockFp8(input, axis, 32, dtype, .f8e8m0), .fp8_block128 => quantizeBlockFp8(input.convert(.bf16), axis, 128, dtype, .f32), @@ -164,23 +187,6 @@ pub fn quantizeMxfp4(x: Tensor, axis: anytype) QuantizedInput { }; } -pub fn quantizeMxfp8(x: Tensor, axis: anytype) QuantizedInput { - const platform = zml.Compiler.current().platform; - - const dtype: ?DataType = switch (platform.target) { - .cuda, .cpu, .tpu => .f8e4m3fn, - .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => .f8e4m3fn, - .cdna3 => .f8e4m3fnuz, - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, - } else null, - else => null, - }; - - if (dtype == null) std.debug.panic("MXFP8 is not supported on {}", .{platform.target}); - return quantizeBlockFp8(x, axis, mx_block_size, dtype.?, .f8e8m0); -} - fn ceilPowerOfTwo(raw_scale: Tensor) Tensor { const bits = raw_scale.bitCast(.u32); return bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 63791ea9..beee2cca 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,228 +2,19 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; -const Dims = Shape.DimsArray; - -const StringBuilder = std.ArrayListUnmanaged(u8); +const testing = @import("testing.zig"); pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); -pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); const log = std.log.scoped(.@"zml/safetensors"); const BYTES_HEADER = 8; -pub const FileType = enum { - index, - safetensors, - unknown, -}; - -pub fn fetchRegistry( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - model: std.Io.File, - model_path: []const u8, -) !TensorRegistry { - const file_type = resolveFiletype(model_path); - - return switch (file_type) { - .index => blk: { - const file_reader_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(file_reader_buf); - - var index_reader = model.reader(io, file_reader_buf); - - var safetensors_index = try parseSafetensorsIndex( - allocator, - &index_reader.interface, - ); - defer safetensors_index.deinit(); - - var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); - - try parseSafetensorsIndexFiles( - allocator, - io, - ®istry, - safetensors_index, - repo, - ); - - break :blk registry; - }, - .safetensors => blk: { - var registry: TensorRegistry = .init(allocator); - - try parseSafetensors(allocator, io, ®istry, model); - - break :blk registry; - }, - else => return error.InvalidPath, - }; -} - -pub const TensorReader = struct { - tensor: Tensor, - file: std.Io.File, - file_reader: std.Io.File.Reader, - alignment: ?std.mem.Alignment, - remaining: u64, - padding_remaining: u64, - - io: std.Io, - interface: std.Io.Reader, - - pub const InitOpts = struct { - alignment: ?std.mem.Alignment = null, - }; - - pub fn init( - io: std.Io, - tensor: Tensor, - buffer: []u8, - opts: InitOpts, - ) !TensorReader { - const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); - errdefer file.close(io); - - const offset, const padding_remaining = if (opts.alignment) |alignment| blk: { - const aligned_offset = std.mem.alignBackward(u64, tensor.offset, alignment.toByteUnits()); - break :blk .{ aligned_offset, tensor.offset - aligned_offset }; - } else blk: { - break :blk .{ tensor.offset, 0 }; - }; - - var file_reader = file.reader(io, buffer); - try file_reader.seekTo(offset); - - return .{ - .tensor = tensor, - .file = file, - .file_reader = file_reader, - .alignment = opts.alignment, - .remaining = tensor.byteSize(), - .padding_remaining = padding_remaining, - .io = io, - .interface = .{ - .vtable = if (opts.alignment) |_| blk: { - break :blk &.{ - .stream = streamAligned, - .discard = discardAligned, - }; - } else blk: { - break :blk &.{ - .stream = stream, - .discard = discard, - }; - }, - .buffer = &.{}, - .seek = 0, - .end = 0, - }, - }; - } - - pub fn deinit(self: *TensorReader) void { - self.file.close(self.io); - } - - // Limit single reads to 1GB to avoid issues on macOS where pread returns - // INT_MAX when the requested size is larger than 2GB - const max_single_read_bytes: u64 = 1 << 30; - - fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const combined_limit = limit.min(.limited64(self.remaining)).min(.limited64(max_single_read_bytes)); - const n = try self.file_reader.interface.stream(w, combined_limit); - self.remaining -= n; - return n; - } - - fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const combined_limit = limit.min(.limited64(self.remaining)); - const n = try self.file_reader.interface.discard(combined_limit); - self.remaining -= n; - return n; - } - - fn streamAligned(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const requested_bytes = @min(limit.minInt64(self.remaining), max_single_read_bytes); - - if (self.padding_remaining > 0) { - _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); - self.padding_remaining = 0; - } - - const align_bytes = self.alignment.?.toByteUnits(); - const file_offset = self.file_reader.pos - std.mem.alignBackward(u64, self.file_reader.pos, align_bytes); - const writer_addr = @intFromPtr(w.buffer.ptr) + w.end; - const writer_offset = writer_addr - std.mem.alignBackward(usize, writer_addr, align_bytes); - - if (file_offset != writer_offset) { - const skip = if (file_offset >= writer_offset) blk: { - break :blk file_offset - writer_offset; - } else blk: { - break :blk align_bytes - writer_offset + file_offset; - }; - - w.advance(skip); - } - - if (w.writableSliceGreedy(align_bytes)) |dest| { - const phys_buf = dest.ptr - file_offset; - const phys_pos = self.file_reader.pos - file_offset; - const phys_size = std.mem.alignBackward(usize, @min(dest.len, requested_bytes) + file_offset, align_bytes); - - if (phys_size > file_offset) { - const phys_read = self.file.readPositional(self.io, &.{phys_buf[0..phys_size]}, phys_pos) catch unreachable; - if (phys_read > file_offset) { - const bytes_read = phys_read - file_offset; - w.advance(bytes_read); - self.file_reader.pos += bytes_read; - self.remaining -= bytes_read; - return bytes_read; - } - } - } else |_| {} - - const bytes_read = try self.file_reader.interface.stream(w, .limited64(requested_bytes)); - self.remaining -= bytes_read; - - return bytes_read; - } - - fn discardAligned(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) return 0; - - if (self.padding_remaining > 0) { - _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); - self.padding_remaining = 0; - } - - const combined_limit = limit.min(.limited64(self.remaining)); - const n = try self.file_reader.interface.discard(combined_limit); - self.remaining -= n; - return n; - } -}; - +/// A checkpoint tensor: its name, its shape, and where its bytes are, at +/// `offset` of the file `file_uri` names. The URI is what the io that read +/// the header resolved, a VFS URI under a VFS, so read the tensor through +/// an io that resolves it alike. A registry's entries own nothing; the +/// registry owns their strings. pub const Tensor = struct { file_uri: []const u8, name: []const u8, @@ -234,8 +25,31 @@ pub const Tensor = struct { return self.shape.byteSize(); } - pub fn reader(self: Tensor, io: std.Io, buffer: []u8, opts: TensorReader.InitOpts) !TensorReader { - return TensorReader.init(io, self, buffer, opts); + /// Reads the whole tensor into `buffer`, which holds exactly its bytes + /// (`OutOfBounds` otherwise). Opens `file_uri` through `io` for the + /// call, failing as that io's open does; a file that ends inside the + /// tensor is `UnexpectedEndOfFile`. + pub fn read(self: Tensor, io: std.Io, buffer: []u8) !void { + if (buffer.len != self.byteSize()) return error.OutOfBounds; + const file = try std.Io.Dir.openFile(.cwd(), io, self.file_uri, .{ .mode = .read_only }); + defer file.close(io); + return self.readRange(io, file, buffer, 0); + } + + pub const RangeError = std.Io.File.ReadPositionalError || error{ OutOfBounds, UnexpectedEndOfFile }; + + /// Reads `buffer.len` bytes of the tensor from `tensor_offset`, relative + /// to its data, out of `file`, which the caller opened at `file_uri`. A + /// range past the tensor is `OutOfBounds`, a file that ends inside it + /// `UnexpectedEndOfFile`. The reads are positional, so concurrent calls + /// may share one file. + pub fn readRange(self: Tensor, io: std.Io, file: std.Io.File, buffer: []u8, tensor_offset: u64) RangeError!void { + const tensor_size = self.byteSize(); + if (tensor_offset > tensor_size or buffer.len > tensor_size - tensor_offset) return error.OutOfBounds; + if (buffer.len == 0) return; + const file_offset = std.math.add(u64, self.offset, tensor_offset) catch return error.OutOfBounds; + _ = std.math.add(u64, file_offset, buffer.len) catch return error.OutOfBounds; + if (try file.readPositionalAll(io, buffer, file_offset) != buffer.len) return error.UnexpectedEndOfFile; } pub fn format(self: Tensor, writer: *std.Io.Writer) !void { @@ -249,212 +63,373 @@ pub const Tensor = struct { } }; +test "Tensor.readRange reads tensor-relative ranges within the tensor and its file" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const file = try tmp.dir.createFile(io, "tensor.bin", .{ .read = true }); + defer file.close(io); + try file.writePositionalAll(io, "prefix0123456789", 0); + + var tensor: Tensor = .{ + .file_uri = "unused", + .name = "tensor", + .shape = .init(.{10}, .u8), + .offset = "prefix".len, + }; + var middle: [4]u8 = undefined; + try tensor.readRange(io, file, &middle, 3); + try std.testing.expectEqualStrings("3456", &middle); + + var empty: [0]u8 = .{}; + try tensor.readRange(io, file, &empty, 10); + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &empty, 11)); + + var too_large: [2]u8 = undefined; + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &too_large, 9)); + + // A file that ends inside the tensor. + tensor.shape = .init(.{12}, .u8); + var tail: [3]u8 = undefined; + try std.testing.expectError(error.UnexpectedEndOfFile, tensor.readRange(io, file, &tail, 9)); + + tensor.offset = std.math.maxInt(u64); + var overflow: [1]u8 = undefined; + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &overflow, 0)); + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &overflow, 1)); +} + +test "Tensor.read reads the whole tensor from its file" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const file = try tmp.dir.createFile(io, "tensor.bin", .{ .read = true }); + try file.writePositionalAll(io, "prefix0123456789", 0); + + var path_buffer: [1024]u8 = undefined; + const path_len = try file.realPath(io, &path_buffer); + file.close(io); + + const tensor: Tensor = .{ + .file_uri = path_buffer[0..path_len], + .name = "tensor", + .shape = .init(.{10}, .u8), + .offset = "prefix".len, + }; + var whole: [10]u8 = undefined; + try tensor.read(io, &whole); + try std.testing.expectEqualStrings("0123456789", &whole); + try std.testing.expectError(error.OutOfBounds, tensor.read(io, whole[0..9])); +} + +/// The tensors of one or more checkpoints by name, read from their +/// safetensors headers; no tensor data is read. It owns every name and URI. pub const TensorRegistry = struct { arena: std.heap.ArenaAllocator, tensors: Tensors, - metadata: Metadatas, - - mutex: std.Io.Mutex = .init, + /// An empty registry. pub fn init(allocator: std.mem.Allocator) TensorRegistry { return .{ .arena = std.heap.ArenaAllocator.init(allocator), .tensors = .{}, - .metadata = .{}, }; } - pub fn initWithMetadata( - allocator: std.mem.Allocator, - metadata: Metadatas, - ) !TensorRegistry { - var self: TensorRegistry = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = .empty, - }; - - try self.mergeMetadata(metadata); - - return self; - } - + /// The checkpoint in `repo`: its `model.safetensors.index.json` and the + /// files it lists, else `model.safetensors`. `repo` stays the caller's + /// to close. pub fn fromRepo( allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, ) !TensorRegistry { - const entrypoint_file, const entrypoint_path = try resolveModelEntrypoint(io, repo); - defer entrypoint_file.close(io); - return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); + const entrypoint, const kind, _ = try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + + var registry: TensorRegistry = .init(allocator); + errdefer registry.deinit(); + try addCheckpoint(io, ®istry, repo, entrypoint, kind, ""); + return registry; } + /// The checkpoint at `path`, read through `io`: a `.safetensors` file, + /// a `.safetensors.index.json` and the files it lists, or a directory + /// holding `model.safetensors.index.json` or `model.safetensors`. A + /// relative path is from the cwd; under a VFS, `path` may be a remote + /// URI. A missing path or entrypoint is `FileNotFound`, a denied + /// directory or entrypoint `PermissionDenied`; a file opened by name, + /// the one given or one an index lists, fails as `io`'s open does (a + /// denied local file is `AccessDenied`). A malformed header, such as + /// one whose offsets leave the file in a truncated download or one + /// shorter than its 8-byte length field, or an index without a + /// `weight_map` from tensor names to file names, is + /// `InvalidSafetensorsHeader`, and a name two of its files repeat + /// `DuplicateTensor`, both logged. pub fn fromPath( allocator: std.mem.Allocator, io: std.Io, path: []const u8, ) !TensorRegistry { - var repo = try resolveModelRepo(io, path); + const repo_path, const file_kind = locateCheckpoint(path); + var repo = try resolveModelRepo(io, repo_path); defer repo.close(io); + const kind = file_kind orelse return fromRepo(allocator, io, repo); - const entrypoint_file, const entrypoint_path = if (std.mem.endsWith(u8, path, ".safetensors.index.json") or std.mem.endsWith(u8, path, ".safetensors")) - .{ try repo.openFile(io, path, .{ .mode = .read_only }), path } - else - try resolveModelEntrypoint(io, repo); - defer entrypoint_file.close(io); + // `repo` is the file's directory: the file opens by its base name. + const entrypoint = try repo.openFile(io, std.fs.path.basename(path), .{ .mode = .read_only }); + defer entrypoint.close(io); - return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); + var registry: TensorRegistry = .init(allocator); + errdefer registry.deinit(); + try addCheckpoint(io, ®istry, repo, entrypoint, kind, ""); + return registry; } + /// Adds every tensor of the checkpoint in `repo`, read as `fromRepo` + /// reads it, under `name` and a dot: `View.withPrefix(name)` reaches + /// them. A second checkpoint, such as a speculative drafter's, joins + /// the registry under a name of its own, and one store then holds both. + /// Adding under a name that a tensor already here is named under fails + /// with `error.DuplicateTensor`, logged, before anything is read. `repo` + /// stays the caller's to close. + pub fn addRepo(self: *TensorRegistry, io: std.Io, repo: std.Io.Dir, name: []const u8) !void { + const prefix = try std.mem.concat(self.arena.child_allocator, u8, &.{ name, "." }); + defer self.arena.child_allocator.free(prefix); + // A checkpoint added beside another owns its prefix: no tensor already + // here may be named under it. + for (self.tensors.keys()) |key| { + if (std.mem.startsWith(u8, key, prefix)) { + log.warn("Cannot add a checkpoint under {s}: tensor {s} is already named under it", .{ name, key }); + return error.DuplicateTensor; + } + } + const entrypoint, const kind, _ = try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + try addCheckpoint(io, self, repo, entrypoint, kind, prefix); + } + + /// Frees every entry, its name and URI. pub fn deinit(self: *TensorRegistry) void { const allocator = self.arena.allocator(); self.tensors.deinit(allocator); - self.metadata.deinit(allocator); self.arena.deinit(); } - pub fn mergeMetadata( - self: *TensorRegistry, - other: Metadatas, - ) !void { - // self.mutex.lock(); - // defer self.mutex.unlock(); - - const allocator = self.arena.allocator(); - - var it = other.iterator(); - while (it.next()) |entry| { - const key = try allocator.dupe(u8, entry.key_ptr.*); - const value = try entry.value_ptr.*.clone(allocator); - - const gop = try self.metadata.getOrPut(allocator, key); - if (gop.found_existing) { - gop.value_ptr.*.deinit(allocator); - gop.value_ptr.* = value; - log.debug("Overwrote existing metadata key={s} with value={f}", .{ key, value }); - } else { - gop.value_ptr.* = value; - log.debug("Added new metadata key={s} with value={f}", .{ key, value }); - } - } - } - + /// Adds `tensor`, its name and file URI copied; a name already here is + /// `DuplicateTensor`, logged. pub fn registerTensor( self: *TensorRegistry, tensor: Tensor, ) !void { const allocator = self.arena.allocator(); - // self.mutex.lock(); - // defer self.mutex.unlock(); - var tensor_copy = tensor; tensor_copy.name = try allocator.dupe(u8, tensor.name); tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); - try self.tensors.put(allocator, tensor_copy.name, tensor_copy); - } - - pub fn reader( - self: *TensorRegistry, - io: std.Io, - tensor_name: []const u8, - buffer: []u8, - ) !TensorReader { - const tensor = self.tensors.get(tensor_name) orelse { - log.err("Tensor {s} not found in registry", .{tensor_name}); - return error.TensorNotFound; - }; - - return try .init(io, tensor, buffer, .{}); - } - - pub fn iterator(self: *TensorRegistry) Tensors.Iterator { - return self.tensors.iterator(); - } - - pub fn totalBytes(self: *TensorRegistry) u64 { - var total: u64 = 0; - - var it = self.tensors.iterator(); - while (it.next()) |entry| { - total += entry.value_ptr.byteSize(); + const entry = try self.tensors.getOrPut(allocator, tensor_copy.name); + if (entry.found_existing) { + log.warn("Tensor {s} of {s} is already registered from {s}", .{ tensor.name, tensor.file_uri, entry.value_ptr.file_uri }); + return error.DuplicateTensor; } - - return total; + entry.value_ptr.* = tensor_copy; } }; +/// What a checkpoint's entrypoint is: an index of the safetensors files +/// holding its tensors, or the one file holding them. +const EntrypointKind = enum { index, safetensors }; + +/// Where the checkpoint at `path` is (`TensorRegistry.fromPath`): its +/// directory, and the kind of the file `path` names there, or null when +/// `path` names the directory itself. +fn locateCheckpoint(path: []const u8) struct { []const u8, ?EntrypointKind } { + const kind: ?EntrypointKind = if (std.mem.endsWith(u8, path, ".safetensors.index.json")) + .index + else if (std.mem.endsWith(u8, path, ".safetensors")) + .safetensors + else + null; + return .{ if (kind == null) path else std.fs.path.dirname(path) orelse ".", kind }; +} + +/// Adds the tensors of the checkpoint whose entrypoint is open in `repo` to +/// `registry` under `prefix`. An index's files are read from `repo`. +fn addCheckpoint( + io: std.Io, + registry: *TensorRegistry, + repo: std.Io.Dir, + entrypoint: std.Io.File, + kind: EntrypointKind, + prefix: []const u8, +) !void { + switch (kind) { + .safetensors => return parseSafetensors(io, registry, entrypoint, prefix), + .index => { + var arena = std.heap.ArenaAllocator.init(registry.arena.child_allocator); + defer arena.deinit(); + const scratch = arena.allocator(); + + var index_uri_buf: [1024]u8 = undefined; + const index_uri = index_uri_buf[0..try entrypoint.realPath(io, &index_uri_buf)]; + + var index_reader = entrypoint.reader(io, try scratch.alloc(u8, 64 * 1024)); + const file_names = indexFiles(scratch, &index_reader.interface) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return index_reader.err orelse err, + else => { + log.warn("Invalid safetensors index {s}: needs a weight_map from tensor names to file names ({t})", .{ index_uri, err }); + return error.InvalidSafetensorsHeader; + }, + }; + for (file_names) |file_name| { + const file = try repo.openFile(io, file_name, .{}); + defer file.close(io); + try parseSafetensors(io, registry, file, prefix); + } + }, + } +} + +/// The files a safetensors index maps tensors to, each once, in the order +/// its `weight_map` first names them, allocated in `arena`. An index +/// without a `weight_map` from tensor names to file names fails with the +/// JSON parser's error, unlogged. +fn indexFiles(arena: std.mem.Allocator, index: *std.Io.Reader) ![]const []const u8 { + var json_reader: std.json.Reader = .init(arena, index); + // The default allocation, `.alloc_always` over a `json.Reader`, copies + // every string out of `index`'s buffer, which the next refill overwrites. + const Index = struct { weight_map: std.json.ArrayHashMap([]const u8) }; + const parsed = try std.json.parseFromTokenSourceLeaky(Index, arena, &json_reader, .{ .ignore_unknown_fields = true }); + var files: std.StringArrayHashMapUnmanaged(void) = .empty; + for (parsed.weight_map.map.values()) |file_name| try files.put(arena, file_name, {}); + return files.keys(); +} + +/// The safetensors files `TensorRegistry.fromPath` reads for `path`, named +/// in their directory, which is returned open for the caller to close: the +/// file a `.safetensors` path names, the files a `.safetensors.index.json` +/// lists, else those the directory's `model.safetensors.index.json` lists, +/// or its `model.safetensors`. The names are allocated in `arena`. No +/// header is read and nothing is logged: a failure here is one `fromPath` +/// reports, with its context. +pub fn checkpointFiles(arena: std.mem.Allocator, io: std.Io, path: []const u8) !struct { std.Io.Dir, []const []const u8 } { + const repo_path, const file_kind = locateCheckpoint(path); + const repo = try std.Io.Dir.openDir(.cwd(), io, repo_path, .{}); + errdefer repo.close(io); + const entrypoint, const kind, const name = if (file_kind) |kind| + .{ try repo.openFile(io, std.fs.path.basename(path), .{ .mode = .read_only }), kind, std.fs.path.basename(path) } + else + try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + const names = switch (kind) { + .safetensors => try arena.dupe([]const u8, &.{name}), + .index => names: { + var index_reader = entrypoint.reader(io, try arena.alloc(u8, 64 * 1024)); + break :names indexFiles(arena, &index_reader.interface) catch |err| switch (err) { + error.ReadFailed => return index_reader.err orelse err, + else => |e| return e, + }; + }, + }; + return .{ repo, names }; +} + +/// Adds the tensors of the safetensors `file` to `registry`, each named +/// `prefix` followed by its key in the header. A header that is not an +/// object of tensors, each with a `dtype`, a `shape` and `data_offsets` +/// spanning exactly its bytes within the file, is +/// `InvalidSafetensorsHeader`, and a name already registered +/// `DuplicateTensor`, both logged. pub fn parseSafetensors( - allocator: std.mem.Allocator, io: std.Io, registry: *TensorRegistry, file: std.Io.File, + prefix: []const u8, ) !void { - var arena = std.heap.ArenaAllocator.init(allocator); + var arena = std.heap.ArenaAllocator.init(registry.arena.child_allocator); defer arena.deinit(); - const header_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(header_buf); - - var reader = file.reader(io, header_buf); + var reader = file.reader(io, try arena.allocator().alloc(u8, 64 * 1024)); var file_uri_buf: [1024]u8 = undefined; const file_uri_len = try file.realPath(io, &file_uri_buf); const file_uri = file_uri_buf[0..file_uri_len]; - const json_header_length: u64 = try reader.interface.takeInt(u64, .little); + // `File.Reader` returns `ReadFailed` with its cause in `err`, which a + // failed seek leaves unset. + const json_header_length: u64 = reader.interface.takeInt(u64, .little) catch |err| switch (err) { + error.EndOfStream => { + log.warn("Invalid safetensors header in {s}: shorter than its {d}-byte header length", .{ file_uri, BYTES_HEADER }); + return error.InvalidSafetensorsHeader; + }, + error.ReadFailed => return reader.err orelse err, + }; + // The header and every tensor's data must lie inside the file. + const file_length = try file.length(io); + if (json_header_length > file_length -| BYTES_HEADER) { + log.warn("Invalid safetensors header in {s}: {d} bytes of header in a file of {d}", .{ file_uri, json_header_length, file_length }); + return error.InvalidSafetensorsHeader; + } const data_start_offset = BYTES_HEADER + json_header_length; const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); - try reader.interface.readSliceAll(json_data); + reader.interface.readSliceAll(json_data) catch |err| return switch (err) { + error.ReadFailed => reader.err orelse err, + error.EndOfStream => err, + }; - const headers: std.json.Value = try std.json.parseFromSliceLeaky( - std.json.Value, - arena.allocator(), - json_data, - .{}, - ); + const headers = std.json.parseFromSliceLeaky(std.json.ArrayHashMap(std.json.Value), arena.allocator(), json_data, .{}) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + else => { + log.warn("Invalid safetensors header in {s}: not a JSON object ({t})", .{ file_uri, err }); + return error.InvalidSafetensorsHeader; + }, + }; + const data_len = file_length - data_start_offset; - var it = headers.object.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; + // `data_offsets` are `[start, end)` after the header. + const Entry = struct { dtype: []const u8, shape: []const i64, data_offsets: [2]u64 }; + var it = headers.map.iterator(); + while (it.next()) |kv| { + const key = kv.key_ptr.*; - if (std.mem.eql(u8, key, "__metadata__")) { - // Parse metadata into a local arena to avoid concurrent allocations - // against registry.arena (which is not thread-safe). - var local_arena = std.heap.ArenaAllocator.init(allocator); - defer local_arena.deinit(); + // Free-form strings, which nothing reads. + if (std.mem.eql(u8, key, "__metadata__")) continue; - const metas = try parseMetadata(local_arena.allocator(), value); - try registry.mergeMetadata(metas); + const entry = std.json.parseFromValueLeaky(Entry, arena.allocator(), kv.value_ptr.*, .{ .ignore_unknown_fields = true }) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + else => { + log.warn("Invalid safetensors header in {s}: tensor {s} needs a string dtype, an integer shape and two unsigned data_offsets ({t})", .{ file_uri, key, err }); + return error.InvalidSafetensorsHeader; + }, + }; - continue; - } - - const shape_field = value.object.get("shape").?.array; - - if (shape_field.items.len > Shape.MAX_RANK) { - log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); + if (entry.shape.len > Shape.MAX_RANK) { + log.warn("Cannot load {s}: tensor {s} has {d} dims, more than {d}", .{ file_uri, key, entry.shape.len, Shape.MAX_RANK }); return error.TooManyDimensions; } + const dtype = stringToDtype(entry.dtype) orelse { + log.warn("Cannot load {s}: tensor {s} has dtype {s}, which zml does not support", .{ file_uri, key, entry.dtype }); + return error.UnsupportedDataType; + }; - const offset_field = value.object.get("data_offsets").?; - const start: u64 = @intCast(offset_field.array.items[0].integer); - const dtype = try stringToDtype(value.object.get("dtype").?.string); - - var dims: Dims = .empty; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); + const start, const end = entry.data_offsets; + const byte_size = checkedByteSize(entry.shape, dtype); + if (byte_size == null or start > end or end > data_len or end - start != byte_size.?) { + log.warn("Invalid safetensors header in {s}: tensor {s} has data_offsets [{d},{d}], which must span the bytes of its {s} shape {any} within the file's {d} data bytes", .{ file_uri, key, start, end, entry.dtype, entry.shape, data_len }); + return error.InvalidSafetensorsHeader; } const tensor: Tensor = .{ .file_uri = file_uri, - .name = key, - .shape = .init(dims.slice(), dtype), + .name = if (prefix.len == 0) key else try std.mem.concat(arena.allocator(), u8, &.{ prefix, key }), + .shape = .init(entry.shape, dtype), .offset = data_start_offset + start, }; @@ -462,294 +437,335 @@ pub fn parseSafetensors( } } -const SafetensorsIndex = struct { - pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); - - arena: std.heap.ArenaAllocator, - - map: Map, - metadata: Metadatas, - - pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .map = .empty, - .metadata = .empty, - }; - } - - pub fn iterator(self: *SafetensorsIndex) Map.Iterator { - return self.map.iterator(); - } - - pub fn deinit(self: *SafetensorsIndex) void { - const allocator = self.arena.allocator(); - self.map.deinit(allocator); - self.metadata.deinit(allocator); - self.arena.deinit(); - } -}; - -fn parseSafetensorsIndex( - allocator: std.mem.Allocator, - reader: *std.Io.Reader, -) !SafetensorsIndex { - var safetensors_index: SafetensorsIndex = .init(allocator); - errdefer safetensors_index.deinit(); - - const arena_allocator = safetensors_index.arena.allocator(); - var json_reader: std.json.Reader = .init(arena_allocator, reader); - - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - arena_allocator, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - const weight_map = index.object.get("weight_map"); - - if (weight_map) |wm| { - var it = wm.object.iterator(); - - while (it.next()) |entry| { - const weight_name = entry.key_ptr.*; - const filename = entry.value_ptr.string; - - const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); - - if (!map_entry.found_existing) { - map_entry.value_ptr.* = .empty; - } - - try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); - } - } else { - log.warn("No weight_map attribute found in index", .{}); - } - - if (index.object.get("metadata")) |metadata_val| { - safetensors_index.metadata = try parseMetadata(arena_allocator, metadata_val); - } - - return safetensors_index; +/// The bytes of `dims` of `dtype`, as `Shape.byteSize` counts them, or null +/// on a negative dim or a bit count past u64. +fn checkedByteSize(dims: []const i64, dtype: DataType) ?u64 { + var bits: u64 = dtype.bitSizeOf(); + for (dims) |d| bits = std.math.mul(u64, bits, std.math.cast(u64, d) orelse return null) catch return null; + return std.math.divCeil(u64, bits, 8) catch unreachable; } -fn parseSafetensorsIndexFiles( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - safetensors_index: SafetensorsIndex, - repo: std.Io.Dir, -) !void { - const filenames = safetensors_index.map.keys(); - for (filenames) |filename| { - const file = try repo.openFile(io, filename, .{}); +test "parseSafetensors rejects a malformed header, data_offsets that do not span their tensor within the file, or a name already registered" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const max = "18446744073709551615"; + // One U8 tensor over the file's 4 data bytes. `header` overrides the + // whole header, `header_len` the header's length field. + const cases = [_]struct { offsets: []const u8 = "[0,4]", shape: []const u8 = "[4]", header: ?[]const u8 = null, header_len: ?u64 = null, valid: bool = false }{ + .{ .valid = true }, + .{ .offsets = "[4,0]" }, + .{ .offsets = "[0,3]" }, + .{ .offsets = "[0,5]" }, + .{ .offsets = "[-4,4]" }, + .{ .offsets = "[0," ++ max ++ "]" }, + .{ .offsets = "[" ++ max ++ "," ++ max ++ "]" }, + .{ .offsets = "[0,18446744073709551616]" }, + .{ .offsets = "[0]" }, + .{ .shape = "[-4]" }, + .{ .shape = "[4294967296,4294967296]" }, + .{ .header = "{\"x\":{\"dtype\":\"U8\",\"shape\":[4]}}" }, + .{ .header = "{\"x\":{\"dtype\":\"U8\",\"shape\":4,\"data_offsets\":[0,4]}}" }, + .{ .header = "[]" }, + .{ .header = "{\"x\":" }, + .{ .header_len = 1 << 20 }, + .{ .header_len = std.math.maxInt(u64) }, + }; + for (cases, 0..) |case, index| { + var header_buffer: [128]u8 = undefined; + const header = case.header orelse try std.fmt.bufPrint(&header_buffer, "{{\"x\":{{\"dtype\":\"U8\",\"shape\":{s},\"data_offsets\":{s}}}}}", .{ case.shape, case.offsets }); + var name_buffer: [32]u8 = undefined; + const file = try tmp.dir.createFile(io, try std.fmt.bufPrint(&name_buffer, "{d}.safetensors", .{index}), .{ .read = true }); defer file.close(io); - try parseSafetensors(allocator, io, registry, file); + var length: [BYTES_HEADER]u8 = undefined; + std.mem.writeInt(u64, &length, case.header_len orelse header.len, .little); + try file.writePositionalAll(io, &length, 0); + try file.writePositionalAll(io, header, length.len); + try file.writePositionalAll(io, &.{ 1, 2, 3, 4 }, length.len + header.len); + + var registry: TensorRegistry = .init(allocator); + defer registry.deinit(); + const result = parseSafetensors(io, ®istry, file, ""); + if (case.valid) { + try result; + try std.testing.expectEqual(@as(u64, BYTES_HEADER + header.len), registry.tensors.get("x").?.offset); + // A name already registered, as one repeated across a checkpoint's + // shards. + try std.testing.expectError(error.DuplicateTensor, parseSafetensors(io, ®istry, file, "")); + } else { + try std.testing.expectError(error.InvalidSafetensorsHeader, result); + } } } -pub fn resolveFiletype(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { - return .safetensors; - } else { - return .unknown; +test "addRepo adds a second checkpoint under a prefix" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // The target is one file, the drafter an index over one shard; both name + // their tensor `w`. + const target = try tmp.dir.createDirPathOpen(io, "target", .{}); + defer target.close(io); + try testing.writeSafetensorsFile(allocator, io, target, "model.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, + }); + const drafter = try tmp.dir.createDirPathOpen(io, "drafter", .{}); + defer drafter.close(io); + try testing.writeSafetensorsFile(allocator, io, drafter, "shard.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 5, 6, 7, 8 } }, + }); + try drafter.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"w\":\"shard.safetensors\"}}" }); + + var registry: TensorRegistry = try .fromRepo(allocator, io, target); + defer registry.deinit(); + try registry.addRepo(io, drafter, "drafter"); + try std.testing.expectError(error.DuplicateTensor, registry.addRepo(io, drafter, "drafter")); + + try std.testing.expectEqual(@as(usize, 2), registry.tensors.count()); + const cases = [_]struct { name: []const u8, bytes: [4]u8 }{ + .{ .name = "w", .bytes = .{ 1, 2, 3, 4 } }, + .{ .name = "drafter.w", .bytes = .{ 5, 6, 7, 8 } }, + }; + for (cases) |case| { + const tensor = registry.tensors.get(case.name).?; + try std.testing.expectEqualStrings(case.name, tensor.name); + var bytes: [4]u8 = undefined; + try tensor.read(io, &bytes); + try std.testing.expectEqualSlices(u8, &case.bytes, &bytes); } } +test "fromPath opens a file, an index or a directory by a path relative to the cwd" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "layer.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, + }); + for ([_][]const u8{ "layer.safetensors.index.json", "model.safetensors.index.json" }) |index| { + try tmp.dir.writeFile(io, .{ .sub_path = index, .data = "{\"weight_map\":{\"w\":\"layer.safetensors\"}}" }); + } + + for ([_][]const u8{ "layer.safetensors", "layer.safetensors.index.json", "." }) |name| { + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); + var registry: TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + var bytes: [4]u8 = undefined; + try registry.tensors.get("w").?.read(io, &bytes); + try std.testing.expectEqualSlices(u8, &.{ 1, 2, 3, 4 }, &bytes); + } +} + +test "fromPath refuses a file shorter than its header length or an index without a weight_map" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const files = [_]struct { []const u8, []const u8 }{ + .{ "short.safetensors", &.{ 1, 2, 3 } }, + .{ "list.safetensors.index.json", "[]" }, + .{ "unmapped.safetensors.index.json", "{\"metadata\":{\"total_size\":4}}" }, + .{ "unnamed.safetensors.index.json", "{\"weight_map\":{\"w\":4}}" }, + }; + for (files) |file| { + const name, const data = file; + try tmp.dir.writeFile(io, .{ .sub_path = name, .data = data }); + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); + try std.testing.expectError(error.InvalidSafetensorsHeader, TensorRegistry.fromPath(allocator, io, path)); + } +} + +test "fromRepo reads every file an index longer than its read buffer names" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const scratch = arena.allocator(); + + // About 90 KiB of weight map, read through a 64 KiB buffer: file names + // straddle refills. + const shards = 16; + const tensors_per_shard = 64; + var index: std.Io.Writer.Allocating = .init(scratch); + try index.writer.writeAll("{\"metadata\":{\"total_size\":1024},\"weight_map\":{"); + for (0..shards) |shard| { + const file_name = try std.fmt.allocPrint(scratch, "model-{d:0>5}-of-{d:0>5}.safetensors", .{ shard + 1, shards }); + var tensors: [tensors_per_shard]testing.TensorBytes = undefined; + for (&tensors, 0..) |*tensor, expert| { + const name = try std.fmt.allocPrint(scratch, "model.layers.{d}.mlp.experts.{d}.down_proj.weight", .{ shard, expert }); + tensor.* = .{ .name = name, .dtype = "U8", .dims = &.{1}, .bytes = &.{0} }; + try index.writer.print("{s}\"{s}\":\"{s}\"", .{ if (shard + expert == 0) "" else ",", name, file_name }); + } + try testing.writeSafetensorsFile(allocator, io, tmp.dir, file_name, &tensors); + } + try index.writer.writeAll("}}"); + try std.testing.expect(index.written().len > 64 * 1024); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = index.written() }); + + var registry: TensorRegistry = try .fromRepo(allocator, io, tmp.dir); + defer registry.deinit(); + // Each file holds its own tensors: a misread file name would lose some. + try std.testing.expectEqual(@as(usize, shards * tensors_per_shard), registry.tensors.count()); +} + +test "checkpointFiles names the files fromPath reads" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const scratch = arena.allocator(); + + // The index names a twice and leaves out c, which sits beside its files + // as a Mistral-format `consolidated.safetensors` does. + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "a.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{1} }, + .{ .name = "x", .dtype = "U8", .dims = &.{1}, .bytes = &.{2} }, + }); + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "b.safetensors", &.{ + .{ .name = "y", .dtype = "U8", .dims = &.{1}, .bytes = &.{3} }, + }); + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "c.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{4} }, + }); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"w\":\"a.safetensors\",\"y\":\"b.safetensors\",\"x\":\"a.safetensors\"}}" }); + const single = try tmp.dir.createDirPathOpen(io, "single", .{}); + defer single.close(io); + try testing.writeSafetensorsFile(allocator, io, single, "model.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{5} }, + }); + + const cases = [_]struct { []const u8, []const []const u8 }{ + .{ ".", &.{ "a.safetensors", "b.safetensors" } }, + .{ "model.safetensors.index.json", &.{ "a.safetensors", "b.safetensors" } }, + .{ "c.safetensors", &.{"c.safetensors"} }, + .{ "single", &.{"model.safetensors"} }, + }; + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + for (cases) |case| { + const name, const expected = case; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); + const repo, const files = try checkpointFiles(scratch, io, path); + defer repo.close(io); + try std.testing.expectEqual(expected.len, files.len); + for (expected, files) |expected_file, file| { + try std.testing.expectEqualStrings(expected_file, file); + (try repo.openFile(io, file, .{})).close(io); + } + // The registry reads these files and no other. + var registry: TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + var read: std.StringArrayHashMapUnmanaged(void) = .empty; + for (registry.tensors.values()) |tensor| try read.put(scratch, std.fs.path.basename(tensor.file_uri), {}); + try std.testing.expectEqual(read.count(), files.len); + for (files) |file| try std.testing.expect(read.contains(file)); + } + const missing = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/missing", .{&tmp.sub_path}); + try std.testing.expectError(error.FileNotFound, checkpointFiles(scratch, io, missing)); +} + pub const ModelPathResolutionError = error{ FileNotFound, + PermissionDenied, InvalidPath, -} || std.mem.Allocator.Error; + Canceled, +}; +/// Opens the model directory at `path`, relative to the cwd. A path that is +/// not a directory, such as a file's, is `InvalidPath`, logged. pub fn resolveModelRepo( io: std.Io, path: []const u8, ) ModelPathResolutionError!std.Io.Dir { - if (std.mem.endsWith(u8, path, ".safetensors.index.json") or - std.mem.endsWith(u8, path, ".safetensors")) - { - const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - - return dir; - } - - const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; + return std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |err| switch (err) { + error.FileNotFound => error.FileNotFound, + error.AccessDenied, error.PermissionDenied => error.PermissionDenied, + error.Canceled => error.Canceled, + else => { + log.warn("{s} is not a model directory: {t}", .{ path, err }); + return error.InvalidPath; + }, }; - - return dir; } -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, []const u8 } { - { - const index_name = "model.safetensors.index.json"; - if (repo.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { - return .{ index_file, "model.safetensors.index.json" }; - } else |_| {} - } +test "resolveModelRepo refuses a file" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors", .data = "" }); - { - const model_name = "model.safetensors"; - if (repo.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { - return .{ model_file, "model.safetensors" }; - } else |_| {} - } - - return ModelPathResolutionError.FileNotFound; + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/model.safetensors", .{&tmp.sub_path}); + try std.testing.expectError(error.InvalidPath, resolveModelRepo(io, path)); } -const Metadata = union(enum) { - null: void, - int: i64, - float: f64, - bool: bool, - string: []const u8, - - array_bool: []const bool, - array_int: []const i64, - array_float: []const f64, - array_string: []const []const u8, - - pub const ItemType = enum { - int, - float, - bool, - string, - - pub fn toZigType(comptime kind: ItemType) type { - return switch (kind) { - .int => i64, - .float => f64, - .bool => bool, - .string => []const u8, - }; - } +/// The entrypoint of the model directory `repo`, open, with its kind and +/// its name there: `model.safetensors.index.json`, else `model.safetensors`. +fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, EntrypointKind, []const u8 } { + // When neither opens, a denied probe is what is reported. On S3 it may + // mean absent: a bucket that may not be listed answers 403 for a + // missing key. + var denied = false; + const entrypoints = [_]struct { []const u8, EntrypointKind }{ + .{ "model.safetensors.index.json", .index }, + .{ "model.safetensors", .safetensors }, }; - - pub fn wrap(x: anytype) Metadata { - return switch (@TypeOf(x)) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, - inline f16, f32, f64 => .{ .float = @floatCast(x) }, - bool => .{ .bool = x }, - []const u8 => .{ .string = x }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), - }; - } - - pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { - return switch (@TypeOf(any_slice[0])) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => { - const res = try allocator.alloc(i64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @intCast(val); - return .{ .array_int = res }; - }, - inline f16, f32, f64 => { - const res = try allocator.alloc(f64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @floatCast(val); - return .{ .array_float = res }; - }, - bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, - []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), - }; - } - - pub fn format( - self: Metadata, - writer: *std.Io.Writer, - ) !void { - switch (self) { - .null => _ = try writer.write("null"), - .string => |s| try writer.print("{s}", .{s}), - .bool => |b| try writer.print("{}", .{b}), - .int => |i| try writer.print("{d}", .{i}), - .float => |f| try writer.print("{d}", .{f}), - .array_bool => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{}", .{v}); - } - try writer.writeByte(']'); - }, - .array_int => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_float => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_string => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("\"{s}\"", .{v}); - } - try writer.writeByte(']'); - }, - } - } - - pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { - switch (self) { - .string => |s| allocator.free(s), - .array_bool => |s| allocator.free(s), - .array_int => |s| allocator.free(s), - .array_float => |s| allocator.free(s), - .array_string => |s| { - for (s) |str| allocator.free(str); - allocator.free(s); - }, + for (entrypoints) |entrypoint| { + const name, const kind = entrypoint; + if (repo.openFile(io, name, .{ .mode = .read_only })) |file| { + return .{ file, kind, name }; + } else |err| switch (err) { + error.Canceled => return ModelPathResolutionError.Canceled, + error.AccessDenied, error.PermissionDenied => denied = true, else => {}, } } + return if (denied) ModelPathResolutionError.PermissionDenied else ModelPathResolutionError.FileNotFound; +} - pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { - return switch (self) { - .null => .null, - .int => |v| .{ .int = v }, - .float => |v| .{ .float = v }, - .bool => |v| .{ .bool = v }, - .string => |s| .{ .string = try allocator.dupe(u8, s) }, - .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, - .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, - .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, - .array_string => |s| blk: { - const new_slice = try allocator.alloc([]const u8, s.len); - errdefer allocator.free(new_slice); - for (s, 0..) |str, i| { - new_slice[i] = try allocator.dupe(u8, str); - } - break :blk .{ .array_string = new_slice }; - }, - }; - } -}; +test "resolveModelEntrypoint falls back past a denied index and reports a denied probe" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const served = try tmp.dir.createDirPathOpen(io, "served", .{}); + defer served.close(io); + try served.writeFile(io, .{ .sub_path = "model.safetensors", .data = "" }); -fn stringToDtype(safetensor_type: []const u8) !DataType { + // The testing io, but a missing file is `PermissionDenied`: S3 answers + // 403 for a missing key to a principal without `s3:ListBucket`. + var vtable = io.vtable.*; + vtable.dirOpenFile = struct { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + return std.testing.io.vtable.dirOpenFile(userdata, dir, sub_path, options) catch |err| switch (err) { + error.FileNotFound => error.PermissionDenied, + else => err, + }; + } + }.dirOpenFile; + const unlisted: std.Io = .{ .userdata = io.userdata, .vtable = &vtable }; + + const file, const kind, const name = try resolveModelEntrypoint(unlisted, served); + defer file.close(io); + try std.testing.expectEqual(.safetensors, kind); + try std.testing.expectEqualStrings("model.safetensors", name); + // `tmp.dir` holds neither entrypoint. + try std.testing.expectError(error.PermissionDenied, resolveModelEntrypoint(unlisted, tmp.dir)); + try std.testing.expectError(error.FileNotFound, resolveModelEntrypoint(io, tmp.dir)); +} + +fn stringToDtype(safetensor_type: []const u8) ?DataType { const map = std.StaticStringMap(DataType).initComptime(.{ .{ "F64", .f64 }, .{ "F32", .f32 }, @@ -769,101 +785,5 @@ fn stringToDtype(safetensor_type: []const u8) !DataType { .{ "BOOL", .bool }, }); - return map.get(safetensor_type) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); - return error.UnsupportedDataType; - }; -} - -fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { - var metadatas: Metadatas = .{}; - var prefix_buf: [1024]u8 = undefined; - var prefix = StringBuilder.initBuffer(&prefix_buf); - - try populateMetadata(allocator, &prefix, val, &metadatas); - - return metadatas; -} - -fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { - const key = prefix.items; - return switch (val) { - .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), - .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), - .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), - .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), - .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), - .array => |v| { - if (v.items.len == 0) return; - if (validSlice(v)) |item_type| { - const data: Metadata = switch (item_type) { - .bool => blk: { - const values = try allocator.alloc(bool, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.bool; - break :blk .{ .array_bool = values }; - }, - .integer => blk: { - const values = try allocator.alloc(i64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.integer; - break :blk .{ .array_int = values }; - }, - .float => blk: { - const values = try allocator.alloc(f64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.float; - break :blk .{ .array_float = values }; - }, - inline .string, .number_string => |tag| blk: { - const values = try allocator.alloc([]const u8, v.items.len); - for (v.items, 0..) |item, i| { - values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); - } - break :blk .{ .array_string = values }; - }, - .null, .array, .object => unreachable, - }; - try metadatas.put(allocator, try allocator.dupe(u8, key), data); - } else { - for (v.items, 0..) |item, i| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try populateMetadata(allocator, prefix, item, metadatas); - prefix.items.len = old_len; - } - } - }, - .object => |v| { - var obj_iter = v.iterator(); - while (obj_iter.next()) |entry| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); - prefix.items.len = old_len; - } - }, - }; -} - -/// We can only create a Zig slice out of json array, if all values -/// in the array have the same type. -fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { - if (v.items.len == 0) return null; - - const item_type: std.meta.Tag(std.json.Value) = v.items[0]; - switch (item_type) { - .null, .array, .object => return null, - else => {}, - } - - for (v.items[1..]) |item| { - if (item != item_type) - return null; - } - - return item_type; + return map.get(safetensor_type); } diff --git a/zml/shape.zig b/zml/shape.zig index b0863367..7f08992e 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -407,6 +407,36 @@ pub const Shape = struct { return self.eql(other) and std.mem.eql(Tag, self.tags(), other.tags()) and self.dtype() == other.dtype(); } + /// Compares dimensions, dtype, tags, mesh, and per-axis partition specs. + pub fn eqlWithTagsAndPartitioning(self: Shape, other: Shape) bool { + if (!self.eqlWithTags(other) or self._sharding.mesh != other._sharding.mesh) return false; + for (0..self.rank()) |ax| { + if (self.partition(ax) != other.partition(ax)) return false; + } + return true; + } + + test eqlWithTagsAndPartitioning { + const other_mesh: Sharding.Mesh = .initTest(.other, .{ .batch, .feature }); + const shape = Shape.init(.{ .out = 8, .in = 16 }, .f32) + .withPartitioning(&test_mesh, .{ .out = .feature, .in = .replicated }); + const Spec = Sharding.Partitioning.Logical.Spec; + // Only the first spec is `shape`'s. + const specs: []const Spec = &.{ .init(.feature), .init(.batch), .replicated, .open }; + for (specs, 0..) |spec, i| { + const other = shape.withPartitioning(&test_mesh, .{ .out = spec, .in = .replicated }); + try testing.expect(shape.eqlWithTags(other)); + try testing.expectEqual(i == 0, shape.eqlWithTagsAndPartitioning(other)); + } + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.withPartitioning(&other_mesh, .{ .out = .feature, .in = .replicated }))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.withDtype(.bf16))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.setDim(.out, 9))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.rename(.{ .out = .features }))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(Shape.init(.{ .out = 8 }, .f32))); + try testing.expect(!shape.withDefaultPartitioning().eqlWithTagsAndPartitioning(Shape.init(.{ .out = 8, .in = 16 }, .f32))); + try testing.expect(Shape.init(.{}, .f32).eqlWithTagsAndPartitioning(Shape.init(.{}, .f32))); + } + /// Format the shape. /// "Shape({.a=10, .b=20}, dtype=.f32)" pub fn format(self: Shape, writer: *std.Io.Writer) !void { @@ -827,9 +857,11 @@ pub const Shape = struct { .grid = undefined, }; + /// Partitions this shape on `mesh` as `Tensor.withPartitioning` takes `partitioning`, except + /// that axes it leaves unspecified are `.replicated`. pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { if (@TypeOf(partitioning) != Sharding.Partitioning) { - return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); + return self.withPartitioning(mesh, Sharding.Partitioning.Logical.init(partitioning).resolve(self.tags(), mesh, .replicated)); } var res = self; res._sharding = .{ .mesh = mesh, .partition = partitioning }; diff --git a/zml/tensor.zig b/zml/tensor.zig index 8c148867..5dd62ddc 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -169,7 +169,9 @@ pub const Tensor = struct { /// Specify the mesh of the input tensor. /// * mesh: *const zml.Mesh, but during compilation the `.meshe_name` syntax can used to get /// a known mesh from the compilation options. - /// * partition spec: a struct where the field names match the axis of the given mesh + /// * partitioning: a `Partitioning`, or anything `Partitioning.Logical.init` accepts: a struct + /// mapping this tensor's axis tags to logical mesh axes, `.replicated` or `.open`. + /// Axes it leaves unspecified are `.open`. /// /// eg `x.withPartitioning(tp, .{ .h = .model }))` or `x.withPartitioning(.tp, .{ .h = .model }))` pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { @@ -179,7 +181,7 @@ pub const Tensor = struct { } if (@TypeOf(partitioning) != Sharding.Partitioning) { - const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, .open, partitioning); + const parsed = Sharding.Partitioning.Logical.init(partitioning).resolve(self._shape.tags(), mesh_, .open); return self.withPartitioning(mesh_, parsed); } @@ -2181,14 +2183,15 @@ pub const Tensor = struct { pub fn absolute(s: Slice, d: i64) Slice { const start = if (s.start < 0) d + s.start else s.start; + // `len` is checked before `end`, whose default would otherwise take the rest of the axis. const end = if (s.singleton) start + 1 + else if (s.len != to_the_end) + start + s.len else if (s.end == to_the_end) d else if (s.end < 0) d + s.end - else if (s.len != to_the_end) - start + s.len else s.end; const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; @@ -2339,6 +2342,7 @@ pub const Tensor = struct { .{ 0, Tensor.Slice{ .end = 1 }, [5]f32{ 0, 1, 2, 3, 4 } }, .{ 1, Tensor.Slice{ .start = 1, .step = 2 }, [4]f32{ 1, 3, 6, 8 } }, .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, + .{ 1, Tensor.Slice{ .start = 1, .len = 2 }, [4]f32{ 1, 2, 6, 7 } }, }) |testcase| { const ax, const slice_, const expectation = testcase; var exe = try zml.module.compile( diff --git a/zml/testing.zig b/zml/testing.zig index 5e3d6333..d1e6faf7 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -57,8 +57,8 @@ fn toSlice(allocator: std.mem.Allocator, io: std.Io, slice_or_buffer: anytype) ! } else .{ slice_or_buffer, false }; } -/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the -/// host for comparison ! +/// Testing utility. Accepts both zml.Buffer and zml.Slice; a zml.Buffer is copied to the host for +/// comparison. pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpts) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; var left: Slice, const should_free_left = try toSlice(allocator, io, left_); @@ -365,20 +365,189 @@ pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpec return error.TestExpectedEqual; } -fn BufferizedWithArgs(comptime T: type) type { - return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); -} +/// `device_count` devices for a loader test: the accelerator the tests run +/// on (`env`, shared and never freed) when it has that many, otherwise a CPU +/// platform with exactly that many, freed by `deinit`. `physical` lays the +/// first `device_count` devices on one axis, so a mesh over it (`mesh`) +/// places on exactly those, whatever the platform has. Must not move after +/// `init`: meshes point at `physical`. +pub const Devices = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + owned: ?*Platform, + physical: Sharding.PhysicalMesh, -fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { - var count: usize = 0; - var it = store.store.registry.iterator(); - while (it.next()) |entry| { - if (std.mem.startsWith(u8, entry.key_ptr.*, prefix)) { - count += 1; - } + pub fn init(self: *Devices, allocator: std.mem.Allocator, io: std.Io, device_count: usize) !void { + const shared = env(); + const owned: ?*Platform = if (shared.target != .cpu and shared.devices.len >= device_count) + null + else + try Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = @intCast(device_count) } }); + errdefer if (owned) |platform| platform.deinit(allocator, io); + const platform: *const Platform = if (owned) |p| p else shared; + const nodes = try allocator.alloc(Sharding.PhysicalNode, device_count); + defer allocator.free(nodes); + for (nodes, platform.devices[0..device_count]) |*node, device| node.* = .device(device); + self.* = .{ + .allocator = allocator, + .io = io, + .platform = platform, + .owned = owned, + .physical = try .fromTree(allocator, platform.target, .axis(.bus, .tree, nodes)), + }; } - return count; + pub fn deinit(self: *Devices) void { + self.physical.deinit(self.allocator); + if (self.owned) |platform| platform.deinit(self.allocator, self.io); + self.* = undefined; + } + + /// A mesh named `name` whose one axis, `model`, spans the devices. + pub fn mesh(self: *const Devices, name: []const u8) !Sharding.Mesh { + return .init(name, &self.physical, .mesh(.{ .model = .high_bandwidth }), .parseBindings(.{ .model = .bus })); + } +}; + +pub const TensorBytes = struct { + name: []const u8, + /// The safetensors dtype name, such as "U16". + dtype: []const u8, + dims: []const i64, + bytes: []const u8, +}; + +/// Writes `tensors` as the safetensors file `sub_path` of `dir`, in order, +/// and registers them from it. +pub fn writeSafetensors( + allocator: std.mem.Allocator, + io: std.Io, + dir: std.Io.Dir, + sub_path: []const u8, + registry: *zml.safetensors.TensorRegistry, + tensors: []const TensorBytes, +) !void { + try writeSafetensorsFile(allocator, io, dir, sub_path, tensors); + const file = try dir.openFile(io, sub_path, .{}); + defer file.close(io); + try zml.safetensors.parseSafetensors(io, registry, file, ""); +} + +/// Writes `tensors` as the safetensors file `sub_path` of `dir`, in order. +pub fn writeSafetensorsFile( + allocator: std.mem.Allocator, + io: std.Io, + dir: std.Io.Dir, + sub_path: []const u8, + tensors: []const TensorBytes, +) !void { + var header: std.Io.Writer.Allocating = .init(allocator); + defer header.deinit(); + const writer = &header.writer; + try writer.writeByte('{'); + var offset: usize = 0; + for (tensors, 0..) |tensor, index| { + if (index != 0) try writer.writeByte(','); + try writer.print("\"{s}\":{{\"dtype\":\"{s}\",\"shape\":[", .{ tensor.name, tensor.dtype }); + for (tensor.dims, 0..) |dim, axis| try writer.print("{s}{d}", .{ if (axis != 0) "," else "", dim }); + try writer.print("],\"data_offsets\":[{d},{d}]}}", .{ offset, offset + tensor.bytes.len }); + offset += tensor.bytes.len; + } + try writer.writeByte('}'); + + const file = try dir.createFile(io, sub_path, .{}); + defer file.close(io); + var header_len: [8]u8 = undefined; + std.mem.writeInt(u64, &header_len, header.written().len, .little); + try file.writePositionalAll(io, &header_len, 0); + var position: u64 = header_len.len; + try file.writePositionalAll(io, header.written(), position); + position += header.written().len; + for (tensors) |tensor| { + try file.writePositionalAll(io, tensor.bytes, position); + position += tensor.bytes.len; + } +} + +/// Distinct, nonzero bytes for test tensor number `seed`. +pub fn pattern(bytes: []u8, seed: u8) void { + for (bytes, 0..) |*byte, index| byte.* = (@as(u8, @truncate(index *% 29)) +% seed *% 53) | 1; +} + +/// A store over a copy of `registry`, for a test that builds several stores +/// over one checkpoint: a store takes its registry's entries. +pub fn storeOf(allocator: std.mem.Allocator, registry: *const zml.safetensors.TensorRegistry, opts: zml.io.TensorStore.Options) !zml.io.TensorStore { + var copy: zml.safetensors.TensorRegistry = .init(allocator); + defer copy.deinit(); + for (registry.tensors.values()) |tensor| try copy.registerTensor(tensor); + return .init(allocator, ©, opts); +} + +/// Loads `model` from `store` with a loader of its own and returns its +/// buffers, which the caller frees with `mem.deinitBufferized`. A recipe +/// for tests: production code creates its loader before building the +/// model, so the loader's set-up runs meanwhile. +pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + model: anytype, + store: *const zml.io.TensorStore, +) !zml.Bufferized(@TypeOf(model.*)) { + var buffers = try zml.mem.bufferize(allocator, model); + errdefer zml.mem.deinitBufferized(allocator, &buffers); + var loader = try zml.io.Loader.init(allocator, io, platform, .{}); + defer loader.deinit(); + try loader.start(model, &buffers, store, .none); + try loader.wait(); + return buffers; +} + +/// Reads every shard of `buffer` back, in its mesh's device order, and +/// checks that it lives in `memory`'s space and holds that device's box of +/// `global`, the whole tensor's row-major bytes (sub-byte elements packed). +/// A buffer without a mesh is replicated on the platform's devices. On +/// CPU/neuron/metal every memory kind is the one memory space, so only +/// CUDA/ROCm/oneAPI tell them apart. +pub fn expectShards(allocator: std.mem.Allocator, io: std.Io, buffer: *const zml.Buffer, global: []const u8, memory: zml.io.TensorStore.Memory) !void { + const platform = buffer._platform; + const shape = platform.placedShape(buffer.shape()).packedShape(); + try std.testing.expectEqual(shape.byteSize(), global.len); + const placement: Sharding.Placement = try .init(&shape); + const devices = placement.mesh().?.devicesInCanonicalOrder(); + try std.testing.expectEqual(devices.len, buffer._shards.len); + const shard_bytes = placement.localShape(shape).byteSize(); + const actual = try allocator.alloc(u8, shard_bytes); + defer allocator.free(actual); + const expected = try allocator.alloc(u8, shard_bytes); + defer allocator.free(expected); + const rank = shape.rank(); + const strides = shape.computeByteStrides(); + var shards = buffer.shards(); + for (devices, 0..) |device, device_index| { + const shard = shards.next().?; + try std.testing.expect(shard._pjrt_buffer.memory(platform.pjrt_api) == platform.devices[device.id].memory(memory.space()).?.pjrt_memory); + try shard.toHost(io, actual); + // The device's box, one run of the last axis per index of the others. + const box = placement.slices(device_index, shape); + const run: usize = if (rank == 0) shape.dtype().sizeOf() else @intCast(box.get(rank - 1).size * strides.get(rank - 1)); + var index: [zml.Shape.MAX_RANK]i64 = @splat(0); + var cursor: usize = 0; + while (cursor < expected.len) : (cursor += run) { + var offset: i64 = 0; + for (0..rank) |axis| offset += (box.get(axis).start + index[axis]) * strides.get(axis); + @memcpy(expected[cursor..][0..run], global[@intCast(offset)..][0..run]); + var axis = rank -| 1; + while (axis > 0) { + axis -= 1; + index[axis] += 1; + if (index[axis] < box.get(axis).size) break; + index[axis] = 0; + } + } + try std.testing.expectEqualSlices(u8, expected, actual); + } } /// Test a layer implementation against reference activations stored in a TensorStore. @@ -419,12 +588,12 @@ pub fn testLayer( const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); const input_count = zml.meta.count(zml.Tensor, &args); - const expected_input_count = countWithPrefix(activation_store, input_name); + const store_input = activation_store.withPrefix(input_name); + const expected_input_count = store_input.count(); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } - const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { @@ -442,7 +611,7 @@ pub fn testLayer( const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); const output_count = exe.output_shapes.len; const store_output = activation_store.withPrefix(output_name); - const expected_output_count = countWithPrefix(store_output, output_name); + const expected_output_count = store_output.count(); if (output_count != expected_output_count) { log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); } @@ -453,34 +622,27 @@ pub fn testLayer( var exe_results = try exe.results(allocator); defer exe_results.deinit(allocator); - var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); - defer zml.meta.visit(struct { - fn cb(_: void, b: *zml.Buffer) void { - b.deinit(); - } - }.cb, {}, &args_buffers); + var args_buffers = try load(allocator, io, platform, &args, activation_store.store); + defer zml.mem.deinitBufferized(allocator, &args_buffers); exe_args.set(.{ layer_weights, args_buffers }); var results = try allocator.alloc(zml.Buffer, output_count); defer allocator.free(results); - exe.call(io, exe_args, &exe_results, .{ .wait = true }); + try exe.call(io, exe_args, &exe_results, .{ .wait = true }); exe_results.fill(.{results}); defer for (results) |*result| result.deinit(); var failed: bool = false; - var reader_buffer: [4096]u8 = undefined; for (0..output_count) |i| { const expected_slice = s: { var buffer: [256]u8 = undefined; const subkey = try std.fmt.bufPrint(&buffer, "{d}", .{i}); - var reader = try store_output.getReader(subkey, io, &reader_buffer); - defer reader.deinit(); - const shape = store_output.getShape(subkey).?; - const expected_slice: zml.Slice = try .alloc(allocator, shape); + const tensor = store_output.get(subkey) orelse return error.TensorNotFound; + const expected_slice: zml.Slice = try .alloc(allocator, tensor.shape); errdefer expected_slice.free(allocator); - try reader.interface.readSliceAll(expected_slice.data()); + try tensor.read(io, expected_slice.data()); break :s expected_slice; }; defer expected_slice.free(allocator); @@ -508,6 +670,39 @@ pub fn testLayer( } } +test testLayer { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = env(); + + const Scale = struct { + weight: zml.Tensor, + + pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { + return x.mul(self.weight); + } + }; + const weight = [_]f32{ 2, -1, 0.5, 3 }; + const input = [_]f32{ 1, 2, 3, 4 }; + const output = [_]f32{ 2, -2, 1.5, 12 }; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: zml.safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try writeSafetensors(allocator, io, tmp.dir, "layer.safetensors", ®istry, &.{ + .{ .name = "scale.weight", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&weight) }, + .{ .name = "x.in.0", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&input) }, + .{ .name = "x.out.0", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&output) }, + }); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); + defer store.deinit(); + + const layer: Scale = .{ .weight = store.view().withPrefix("scale").createReplicatedTensor("weight", null) }; + var layer_weights = try load(allocator, io, platform, &layer, &store); + defer zml.mem.deinitBufferized(allocator, &layer_weights); + try testLayer(allocator, io, platform, layer, .forward, store.view(), "x", layer_weights, &.{}, .{}); +} + fn center(slice: anytype, i: usize) @TypeOf(slice) { const c = 20; const start = if (i < c) 0 else i - c; @@ -518,3 +713,27 @@ fn center(slice: anytype, i: usize) @TypeOf(slice) { pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { return std.testing.expectEqual(expected, actual); } + +test load { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: zml.safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var bytes: [8]u8 = undefined; + pattern(&bytes, 1); + try writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "weight", .dtype = "U8", .dims = &.{8}, .bytes = &bytes }, + }); + var store = try storeOf(allocator, ®istry, .{}); + defer store.deinit(); + // The store took a copy: the registry still holds its entries. + try std.testing.expectEqual(1, registry.tensors.count()); + + const Model = struct { weight: zml.Tensor }; + const model: Model = .{ .weight = store.view().createReplicatedTensor("weight", null) }; + var buffers = try load(allocator, io, env(), &model, &store); + defer zml.mem.deinitBufferized(allocator, &buffers); + try expectShards(allocator, io, &buffers.weight, &bytes, .default); +}